t-dna-ngrams-Multilingual.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# T-DNA\n",
    "https://github.com/shizhediao/T-DNA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This notebook generates ngrams for training using the T-DNA architecture\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.10.2+cu102'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'vocabulary_utils' from '/Users/kylehamilton/MyDocuments/ML-Labs/kinzen/projects/TAPT-n/vocabulary_utils.py'>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "import fasttext\n",
    "import spacy\n",
    "import json\n",
    "import csv\n",
    "import matplotlib.pyplot as plt\n",
    "import vocabulary_utils as vu\n",
    "\n",
    "import importlib\n",
    "importlib.reload(vu)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import vocabulary from xlm-roberta\n",
    "We use this to remove ngrams which are already in the vocabulary.\n",
    "\n",
    "If you want to use this technique with a different model, just make sure to fetch that model's vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary size: 250000\n"
     ]
    }
   ],
   "source": [
    "with open('../models/xlm-roberta-large/vocab.json') as json_file:\n",
    "    data = json.load(json_file)\n",
    "    print(\"Vocabulary size:\",len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# xlm-roberta uses sentencepiece bpe which uses undersacores for spaces to play nice with multiple languages.\n",
    "# in order to match words in the xlm-roberta vocab, we first get rid of the underscores.\n",
    "xlm_roberta_vocab = [s.replace('▁','') for s in list(data.keys())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<unk>', '<s>', '</s>', ',', '.', '', 's', 'de', '-', 'a']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xlm_roberta_vocab[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n",
      "True\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "# a couple of checks for good measure...\n",
    "print('vaccine' in xlm_roberta_vocab)\n",
    "print('vac' in xlm_roberta_vocab)\n",
    "print('cine' in xlm_roberta_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "False\n"
     ]
    }
   ],
   "source": [
    "print('France' in xlm_roberta_vocab)\n",
    "print('france' in xlm_roberta_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>path</th>\n",
       "      <th>model</th>\n",
       "      <th>use_stopwords</th>\n",
       "      <th>language</th>\n",
       "      <th>stopwords</th>\n",
       "      <th>additionalInfo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1171</td>\n",
       "      <td>audio/audio_ar-SA.txt</td>\n",
       "      <td>camel_tools</td>\n",
       "      <td>1</td>\n",
       "      <td>arabic</td>\n",
       "      <td>ar-stop-words.txt</td>\n",
       "      <td>https://towardsdatascience.com/arabic-nlp-uniq...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8926</td>\n",
       "      <td>audio/audio_de-DE.txt</td>\n",
       "      <td>de_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>german</td>\n",
       "      <td>stopwords-de.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>34073</td>\n",
       "      <td>audio/audio_en-US.txt</td>\n",
       "      <td>en_core_web_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>english</td>\n",
       "      <td>stopwords-en.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>129</td>\n",
       "      <td>audio/audio_es-ES.txt</td>\n",
       "      <td>es_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>spanish</td>\n",
       "      <td>stopwords-es.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2690</td>\n",
       "      <td>audio/audio_es-MX.txt</td>\n",
       "      <td>es_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>spanish</td>\n",
       "      <td>stopwords-es.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>672</td>\n",
       "      <td>audio/audio_fr-FR.txt</td>\n",
       "      <td>fr_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>french</td>\n",
       "      <td>stopwords-fr.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1050</td>\n",
       "      <td>audio/audio_hi-IN.txt</td>\n",
       "      <td>en_core_web_sm</td>\n",
       "      <td>1</td>\n",
       "      <td>hindi</td>\n",
       "      <td>hi-stop-words.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1240</td>\n",
       "      <td>audio/audio_pt-BR.txt</td>\n",
       "      <td>pt_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>portuguese</td>\n",
       "      <td>stopwords-pt.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1250</td>\n",
       "      <td>audio/audio_ru-RU.txt</td>\n",
       "      <td>ru_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>russian</td>\n",
       "      <td>stopwords-ru.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1585</td>\n",
       "      <td>audio/audio_sv-SE.txt</td>\n",
       "      <td>xx_sent_ud_sm</td>\n",
       "      <td>1</td>\n",
       "      <td>swedish</td>\n",
       "      <td>sv-stop-words.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>4277</td>\n",
       "      <td>audio/audio_tr-TR.txt</td>\n",
       "      <td>en_core_web_sm</td>\n",
       "      <td>1</td>\n",
       "      <td>turkish</td>\n",
       "      <td>tr-stop-words.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>34244</td>\n",
       "      <td>video/video_de-DE.txt</td>\n",
       "      <td>de_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>german</td>\n",
       "      <td>stopwords-de.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>999</td>\n",
       "      <td>video/video_en-US.txt</td>\n",
       "      <td>en_core_web_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>english</td>\n",
       "      <td>stopwords-en.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>396</td>\n",
       "      <td>video/video_fr-FR.txt</td>\n",
       "      <td>fr_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>french</td>\n",
       "      <td>stopwords-fr.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1</td>\n",
       "      <td>video/video_kz.txt</td>\n",
       "      <td>en_core_web_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>english</td>\n",
       "      <td>stopwords-en.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>3864</td>\n",
       "      <td>video/video_pt-BR.txt</td>\n",
       "      <td>pt_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>portuguese</td>\n",
       "      <td>stopwords-pt.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>135</td>\n",
       "      <td>video/video_ru.txt</td>\n",
       "      <td>ru_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>russian</td>\n",
       "      <td>stopwords-ru.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>3</td>\n",
       "      <td>video/video_sv-SE.txt</td>\n",
       "      <td>xx_sent_ud_sm</td>\n",
       "      <td>1</td>\n",
       "      <td>swedish</td>\n",
       "      <td>sv-stop-words.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>5</td>\n",
       "      <td>video/video_ua.txt</td>\n",
       "      <td>en_core_web_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>english</td>\n",
       "      <td>stopwords-en.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>10</td>\n",
       "      <td>video/video_us.txt</td>\n",
       "      <td>en_core_web_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>english</td>\n",
       "      <td>stopwords-en.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>8500</td>\n",
       "      <td>video/video_zh-CN.txt</td>\n",
       "      <td>zh_core_web_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>chinese</td>\n",
       "      <td>stopwords-zh.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>10842</td>\n",
       "      <td>video/video_zh-HK.txt</td>\n",
       "      <td>zh_core_web_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>chinese</td>\n",
       "      <td>stopwords-zh.txt</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    count                   path            model  use_stopwords    language  \\\n",
       "0    1171  audio/audio_ar-SA.txt      camel_tools              1      arabic   \n",
       "1    8926  audio/audio_de-DE.txt  de_core_news_sm              0      german   \n",
       "2   34073  audio/audio_en-US.txt   en_core_web_sm              0     english   \n",
       "3     129  audio/audio_es-ES.txt  es_core_news_sm              0     spanish   \n",
       "4    2690  audio/audio_es-MX.txt  es_core_news_sm              0     spanish   \n",
       "5     672  audio/audio_fr-FR.txt  fr_core_news_sm              0      french   \n",
       "6    1050  audio/audio_hi-IN.txt   en_core_web_sm              1       hindi   \n",
       "7    1240  audio/audio_pt-BR.txt  pt_core_news_sm              0  portuguese   \n",
       "8    1250  audio/audio_ru-RU.txt  ru_core_news_sm              0     russian   \n",
       "9    1585  audio/audio_sv-SE.txt    xx_sent_ud_sm              1     swedish   \n",
       "10   4277  audio/audio_tr-TR.txt   en_core_web_sm              1     turkish   \n",
       "11  34244  video/video_de-DE.txt  de_core_news_sm              0      german   \n",
       "12    999  video/video_en-US.txt   en_core_web_sm              0     english   \n",
       "13    396  video/video_fr-FR.txt  fr_core_news_sm              0      french   \n",
       "14      1     video/video_kz.txt   en_core_web_sm              0     english   \n",
       "15   3864  video/video_pt-BR.txt  pt_core_news_sm              0  portuguese   \n",
       "16    135     video/video_ru.txt  ru_core_news_sm              0     russian   \n",
       "17      3  video/video_sv-SE.txt    xx_sent_ud_sm              1     swedish   \n",
       "18      5     video/video_ua.txt   en_core_web_sm              0     english   \n",
       "19     10     video/video_us.txt   en_core_web_sm              0     english   \n",
       "20   8500  video/video_zh-CN.txt   zh_core_web_sm              0     chinese   \n",
       "21  10842  video/video_zh-HK.txt   zh_core_web_sm              0     chinese   \n",
       "\n",
       "            stopwords                                     additionalInfo  \n",
       "0   ar-stop-words.txt  https://towardsdatascience.com/arabic-nlp-uniq...  \n",
       "1    stopwords-de.txt                                                NaN  \n",
       "2    stopwords-en.txt                                                NaN  \n",
       "3    stopwords-es.txt                                                NaN  \n",
       "4    stopwords-es.txt                                                NaN  \n",
       "5    stopwords-fr.txt                                                NaN  \n",
       "6   hi-stop-words.txt                                                NaN  \n",
       "7    stopwords-pt.txt                                                NaN  \n",
       "8    stopwords-ru.txt                                                NaN  \n",
       "9   sv-stop-words.txt                                                NaN  \n",
       "10  tr-stop-words.txt                                                NaN  \n",
       "11   stopwords-de.txt                                                NaN  \n",
       "12   stopwords-en.txt                                                NaN  \n",
       "13   stopwords-fr.txt                                                NaN  \n",
       "14   stopwords-en.txt                                                NaN  \n",
       "15   stopwords-pt.txt                                                NaN  \n",
       "16   stopwords-ru.txt                                                NaN  \n",
       "17  sv-stop-words.txt                                                NaN  \n",
       "18   stopwords-en.txt                                                NaN  \n",
       "19   stopwords-en.txt                                                NaN  \n",
       "20   stopwords-zh.txt                                                NaN  \n",
       "21   stopwords-zh.txt                                                NaN  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transcripts_metadata = pd.read_csv('../data/all-transcripts/transcripts.tsv',sep=\"\\t\")\n",
    "transcripts_metadata"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "SpaCy xx_sent_ud_sm\n",
    "Universal Dependencies v2.8\n",
    "    UD_Afrikaans-AfriBooms,\n",
    "    UD_Croatian-SET,\n",
    "    UD_Czech-CAC,\n",
    "    UD_Czech-CLTT,\n",
    "    UD_Danish-DDT,\n",
    "    UD_Dutch-Alpino,\n",
    "    UD_Dutch-LassySmall,\n",
    "    UD_English-EWT,\n",
    "    UD_Finnish-FTB,\n",
    "    UD_Finnish-TDT,\n",
    "    UD_French-GSD,\n",
    "    UD_French-Spoken,\n",
    "    UD_German-GSD,\n",
    "    UD_Indonesian-GSD,\n",
    "    UD_Irish-IDT,\n",
    "    UD_Italian-TWITTIRO,\n",
    "    UD_Korean-GSD,\n",
    "    UD_Korean-Kaist,\n",
    "    UD_Latvian-LVTB,\n",
    "    UD_Lithuanian-ALKSNIS,\n",
    "    UD_Lithuanian-HSE,\n",
    "    UD_Marathi-UFAL,\n",
    "    UD_Norwegian-Bokmaal,\n",
    "    UD_Norwegian-Nynorsk,\n",
    "    UD_Norwegian-NynorskLIA,\n",
    "    UD_Persian-Seraji,\n",
    "    UD_Portuguese-Bosque,\n",
    "    UD_Portuguese-GSD,\n",
    "    UD_Romanian-Nonstandard,\n",
    "    UD_Romanian-RRT,\n",
    "    UD_Russian-GSD,\n",
    "    UD_Russian-Taiga,\n",
    "    UD_Serbian-SET,\n",
    "    UD_Slovak-SNK,\n",
    "    UD_Spanish-GSD,\n",
    "    UD_Swedish-Talbanken,\n",
    "    UD_Telugu-MTG,\n",
    "    UD_Vietnamese-VTB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['german',\n",
       " 'hindi',\n",
       " 'russian',\n",
       " 'chinese',\n",
       " 'english',\n",
       " 'spanish',\n",
       " 'french',\n",
       " 'arabic',\n",
       " 'swedish',\n",
       " 'portuguese',\n",
       " 'turkish']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "languages = list(set(transcripts_metadata['language']))\n",
    "languages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['fr_core_news_sm',\n",
       " 'xx_sent_ud_sm',\n",
       " 'zh_core_web_sm',\n",
       " 'pt_core_news_sm',\n",
       " 'de_core_news_sm',\n",
       " 'ru_core_news_sm',\n",
       " 'camel_tools',\n",
       " 'es_core_news_sm',\n",
       " 'en_core_web_sm']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "models = list(set(transcripts_metadata['model']))\n",
    "models"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "source": [
    "# Download all models upfront for convenience\n",
    "for model in models:\n",
    "    try:\n",
    "        !python -m spacy download {model}\n",
    "    except:\n",
    "        print(\"not a spacy model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Arabic ngrams\n",
    "Arabic is special and has its own tokenization libraries. See this blogpost: https://towardsdatascience.com/arabic-nlp-unique-challenges-and-their-solutions-d99e8a87893d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting camel_tools\n",
      "  Downloading camel_tools-1.2.0.tar.gz (58 kB)\n",
      "     |████████████████████████████████| 58 kB 1.8 MB/s             \n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: future in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from camel_tools) (0.18.2)\n",
      "Requirement already satisfied: six in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from camel_tools) (1.15.0)\n",
      "Requirement already satisfied: docopt in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from camel_tools) (0.6.2)\n",
      "Collecting cachetools\n",
      "  Downloading cachetools-4.2.4-py3-none-any.whl (10 kB)\n",
      "Requirement already satisfied: numpy in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from camel_tools) (1.19.5)\n",
      "Requirement already satisfied: scipy in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from camel_tools) (1.5.3)\n",
      "Requirement already satisfied: pandas in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from camel_tools) (1.1.5)\n",
      "Requirement already satisfied: scikit-learn in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from camel_tools) (0.24.1)\n",
      "Requirement already satisfied: dill in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from camel_tools) (0.3.4)\n",
      "Collecting torch>=1.3\n",
      "  Downloading torch-1.10.2-cp36-cp36m-manylinux1_x86_64.whl (881.9 MB)\n",
      "     |████████████▋                   | 347.1 MB 156.2 MB/s eta 0:00:04"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     |██████████████████████████▎     | 725.1 MB 167.0 MB/s eta 0:00:01"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     |███████████████████████████████ | 854.7 MB 167.3 MB/s eta 0:00:01"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     |████████████████████████████████| 881.9 MB 7.9 kB/s              \n",
      "\u001b[?25hCollecting transformers>=3.0.2\n",
      "  Downloading transformers-4.17.0-py3-none-any.whl (3.8 MB)\n",
      "     |████████████████████████████████| 3.8 MB 82.4 MB/s            \n",
      "\u001b[?25hCollecting editdistance\n",
      "  Downloading editdistance-0.6.0-cp36-cp36m-manylinux2010_x86_64.whl (284 kB)\n",
      "     |████████████████████████████████| 284 kB 144.8 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: requests in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from camel_tools) (2.26.0)\n",
      "Collecting camel-kenlm\n",
      "  Downloading camel-kenlm-2021.12.27.tar.gz (418 kB)\n",
      "     |████████████████████████████████| 418 kB 144.0 MB/s            \n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: typing-extensions in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from torch>=1.3->camel_tools) (3.10.0.2)\n",
      "Requirement already satisfied: dataclasses in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from torch>=1.3->camel_tools) (0.8)\n",
      "Collecting huggingface-hub<1.0,>=0.1.0\n",
      "  Downloading huggingface_hub-0.4.0-py3-none-any.whl (67 kB)\n",
      "     |████████████████████████████████| 67 kB 11.6 MB/s             \n",
      "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from transformers>=3.0.2->camel_tools) (2020.11.13)\n",
      "Requirement already satisfied: importlib-metadata in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from transformers>=3.0.2->camel_tools) (3.6.0)\n",
      "Collecting sacremoses\n",
      "  Downloading sacremoses-0.0.47-py2.py3-none-any.whl (895 kB)\n",
      "     |████████████████████████████████| 895 kB 25.3 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from transformers>=3.0.2->camel_tools) (5.4.1)\n",
      "Requirement already satisfied: filelock in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from transformers>=3.0.2->camel_tools) (3.0.12)\n",
      "Requirement already satisfied: tqdm>=4.27 in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from transformers>=3.0.2->camel_tools) (4.62.3)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from transformers>=3.0.2->camel_tools) (21.3)\n",
      "Collecting tokenizers!=0.11.3,>=0.11.1\n",
      "  Downloading tokenizers-0.11.6-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.5 MB)\n",
      "     |████████████████████████████████| 6.5 MB 88.3 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: python-dateutil>=2.7.3 in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from pandas->camel_tools) (2.8.1)\n",
      "Requirement already satisfied: pytz>=2017.2 in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from pandas->camel_tools) (2021.1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from requests->camel_tools) (1.26.7)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from requests->camel_tools) (2.0.9)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from requests->camel_tools) (3.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from requests->camel_tools) (2021.5.30)\n",
      "Requirement already satisfied: joblib>=0.11 in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from scikit-learn->camel_tools) (1.0.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from scikit-learn->camel_tools) (2.1.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from packaging>=20.0->transformers>=3.0.2->camel_tools) (2.4.7)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from importlib-metadata->transformers>=3.0.2->camel_tools) (3.4.0)\n",
      "Requirement already satisfied: click in /home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages (from sacremoses->transformers>=3.0.2->camel_tools) (7.1.2)\n",
      "Building wheels for collected packages: camel-tools, camel-kenlm\n",
      "  Building wheel for camel-tools (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for camel-tools: filename=camel_tools-1.2.0-py3-none-any.whl size=99034 sha256=ab26b2cb9ac8789278baab1f656f30b26c1b4db62d62ba72025218b82c23ad2e\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/a5/0a/03/03a65f710702aa79fa6db59d0177ca164512797707ae2106dd\n",
      "  Building wheel for camel-kenlm (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for camel-kenlm: filename=camel_kenlm-2021.12.27-cp36-cp36m-linux_x86_64.whl size=2111732 sha256=1e305744543625e401056772268fe2d86902de91ca80db055306807e39418a3a\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/76/20/f8/e86181a6e99a7df21f499a2cd21a42e086a0f9fcd58a551cce\n",
      "Successfully built camel-tools camel-kenlm\n",
      "Installing collected packages: tokenizers, sacremoses, huggingface-hub, transformers, torch, editdistance, camel-kenlm, cachetools, camel-tools\n",
      "Successfully installed cachetools-4.2.4 camel-kenlm-2021.12.27 camel-tools-1.2.0 editdistance-0.6.0 huggingface-hub-0.4.0 sacremoses-0.0.47 tokenizers-0.11.6 torch-1.10.2 transformers-4.17.0\n"
     ]
    }
   ],
   "source": [
    "!pip install camel_tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\"عناوين اليوم أول زفاف في ألم تبرز طيران الإمارات تتوقع زيادة عدد مسافريها إلى واحد فصل 01,000,000. '"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_to_data = \"../data/transcripts/audio_ar-SA.txt\"\n",
    "ar_df = pd.read_csv(path_to_data,header=None,names=['text'])\n",
    "ar_df['text'][0][:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import the dediacritization tool\n",
    "from camel_tools.utils.dediac import dediac_ar\n",
    "from camel_tools.utils.normalize import normalize_alef_maksura_ar\n",
    "from camel_tools.utils.normalize import normalize_alef_ar\n",
    "from camel_tools.utils.normalize import normalize_teh_marbuta_ar\n",
    "from camel_tools.tokenizers.word import simple_word_tokenize\n",
    "\n",
    "# apply to your text column\n",
    "ar_df['text'] = ar_df['text'].apply(dediac_ar)\n",
    "\n",
    "def ortho_normalize(text):\n",
    "    text = normalize_alef_maksura_ar(text)\n",
    "    text = normalize_alef_ar(text)\n",
    "    text = normalize_teh_marbuta_ar(text)\n",
    "    return text\n",
    "  \n",
    "ar_df['text'] = ar_df['text'].apply(ortho_normalize)\n",
    "ar_df['text'] = ar_df['text'].apply(simple_word_tokenize)\n",
    "\n",
    "\n",
    "\n",
    "ar_df.to_csv('../data/transcripts/transcripts-ar-pretokenized.csv',header=None,index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"\n",
      "عناوين\n",
      "اليوم\n",
      "اول\n",
      "زفاف\n",
      "في\n",
      "الم\n",
      "تبرز\n",
      "طيران\n",
      "الامارات\n"
     ]
    }
   ],
   "source": [
    "for d in ar_df['text'][0][:10]:\n",
    "    print(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "!camel light"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'diac': 'فَصَلَ', 'lex': 'فَصَل', 'bw': 'فَصَل/PV+َ/PVSUFF_SUBJ:3MS', 'gloss': 'separate;detach;set_apart+he;it_<verb>', 'pos': 'verb', 'prc3': '0', 'prc2': '0', 'prc1': '0', 'prc0': '0', 'per': '3', 'asp': 'p', 'vox': 'a', 'mod': 'i', 'stt': 'na', 'cas': 'na', 'enc0': '0', 'rat': 'n', 'source': 'lex', 'form_gen': 'm', 'form_num': 's', 'd3seg': 'فَصَلَ', 'caphi': 'f_a_s._a_l_a', 'd1tok': 'فَصَلَ', 'd2tok': 'فَصَلَ', 'pos_logprob': -1.023208, 'd3tok': 'فَصَلَ', 'd2seg': 'فَصَلَ', 'pos_lex_logprob': -4.497461, 'num': 's', 'ud': 'VERB', 'gen': 'm', 'catib6': 'VRB', 'root': 'ف.ص.ل', 'bwtok': 'فَصَل_+َ', 'pattern': '1َ2َ3َ', 'lex_logprob': -4.497461, 'atbtok': 'فَصَلَ', 'atbseg': 'فَصَلَ', 'd1seg': 'فَصَلَ', 'stem': 'فَصَل', 'stemgloss': 'separate;detach;set_apart', 'stemcat': 'PV'}\n"
     ]
    }
   ],
   "source": [
    "from camel_tools.morphology.database import MorphologyDB\n",
    "from camel_tools.morphology.analyzer import Analyzer\n",
    "\n",
    "db = MorphologyDB.builtin_db()\n",
    "analyzer = Analyzer(db)\n",
    "\n",
    "analyses = analyzer.analyze('فصل')\n",
    "\n",
    "print(analyses[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chinese\n",
    "Chinese is also special since it causes some funny runtime errors. We are omitting it in phase 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pretokenize chinese\n",
    "def pretokenize(text):\n",
    "    nlp = spacy.load('zh_core_web_sm')\n",
    "    _text = nlp(str(text))\n",
    "    tokens=[]\n",
    "    for token in _text: \n",
    "        if not token.is_punct and not token.is_stop:\n",
    "            tokens.append(token.text)\n",
    "    return tokens\n",
    "\n",
    "chinese['text'] = chinese['text'].apply(pretokenize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chinese.to_csv('../data/transcripts/transcripts-zh-pretokenized.csv',header=None,index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "source": [
    "# Multilingual ngrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "importlib.reload(vu)\n",
    "\n",
    "# get ngrams for all language except arabic, and chinese\n",
    "all_transcripts = transcripts_metadata[(transcripts_metadata['language']!='arabic') & (transcripts_metadata['language']!='chinese')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_transcripts = all_transcripts.sort_values(by=\"language\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_transcripts.fillna('', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>path</th>\n",
       "      <th>model</th>\n",
       "      <th>use_stopwords</th>\n",
       "      <th>language</th>\n",
       "      <th>stopwords</th>\n",
       "      <th>additionalInfo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>10</td>\n",
       "      <td>video/video_us.txt</td>\n",
       "      <td>en_core_web_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>english</td>\n",
       "      <td>stopwords-en.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>34073</td>\n",
       "      <td>audio/audio_en-US.txt</td>\n",
       "      <td>en_core_web_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>english</td>\n",
       "      <td>stopwords-en.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1</td>\n",
       "      <td>video/video_kz.txt</td>\n",
       "      <td>en_core_web_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>english</td>\n",
       "      <td>stopwords-en.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>999</td>\n",
       "      <td>video/video_en-US.txt</td>\n",
       "      <td>en_core_web_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>english</td>\n",
       "      <td>stopwords-en.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>5</td>\n",
       "      <td>video/video_ua.txt</td>\n",
       "      <td>en_core_web_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>english</td>\n",
       "      <td>stopwords-en.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>396</td>\n",
       "      <td>video/video_fr-FR.txt</td>\n",
       "      <td>fr_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>french</td>\n",
       "      <td>stopwords-fr.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>672</td>\n",
       "      <td>audio/audio_fr-FR.txt</td>\n",
       "      <td>fr_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>french</td>\n",
       "      <td>stopwords-fr.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>34244</td>\n",
       "      <td>video/video_de-DE.txt</td>\n",
       "      <td>de_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>german</td>\n",
       "      <td>stopwords-de.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8926</td>\n",
       "      <td>audio/audio_de-DE.txt</td>\n",
       "      <td>de_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>german</td>\n",
       "      <td>stopwords-de.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1050</td>\n",
       "      <td>audio/audio_hi-IN.txt</td>\n",
       "      <td>en_core_web_sm</td>\n",
       "      <td>1</td>\n",
       "      <td>hindi</td>\n",
       "      <td>hi-stop-words.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1240</td>\n",
       "      <td>audio/audio_pt-BR.txt</td>\n",
       "      <td>pt_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>portuguese</td>\n",
       "      <td>stopwords-pt.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>3864</td>\n",
       "      <td>video/video_pt-BR.txt</td>\n",
       "      <td>pt_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>portuguese</td>\n",
       "      <td>stopwords-pt.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1250</td>\n",
       "      <td>audio/audio_ru-RU.txt</td>\n",
       "      <td>ru_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>russian</td>\n",
       "      <td>stopwords-ru.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>135</td>\n",
       "      <td>video/video_ru.txt</td>\n",
       "      <td>ru_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>russian</td>\n",
       "      <td>stopwords-ru.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2690</td>\n",
       "      <td>audio/audio_es-MX.txt</td>\n",
       "      <td>es_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>spanish</td>\n",
       "      <td>stopwords-es.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>129</td>\n",
       "      <td>audio/audio_es-ES.txt</td>\n",
       "      <td>es_core_news_sm</td>\n",
       "      <td>0</td>\n",
       "      <td>spanish</td>\n",
       "      <td>stopwords-es.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>3</td>\n",
       "      <td>video/video_sv-SE.txt</td>\n",
       "      <td>xx_sent_ud_sm</td>\n",
       "      <td>1</td>\n",
       "      <td>swedish</td>\n",
       "      <td>sv-stop-words.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1585</td>\n",
       "      <td>audio/audio_sv-SE.txt</td>\n",
       "      <td>xx_sent_ud_sm</td>\n",
       "      <td>1</td>\n",
       "      <td>swedish</td>\n",
       "      <td>sv-stop-words.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>4277</td>\n",
       "      <td>audio/audio_tr-TR.txt</td>\n",
       "      <td>en_core_web_sm</td>\n",
       "      <td>1</td>\n",
       "      <td>turkish</td>\n",
       "      <td>tr-stop-words.txt</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    count                   path            model  use_stopwords    language  \\\n",
       "19     10     video/video_us.txt   en_core_web_sm              0     english   \n",
       "2   34073  audio/audio_en-US.txt   en_core_web_sm              0     english   \n",
       "14      1     video/video_kz.txt   en_core_web_sm              0     english   \n",
       "12    999  video/video_en-US.txt   en_core_web_sm              0     english   \n",
       "18      5     video/video_ua.txt   en_core_web_sm              0     english   \n",
       "13    396  video/video_fr-FR.txt  fr_core_news_sm              0      french   \n",
       "5     672  audio/audio_fr-FR.txt  fr_core_news_sm              0      french   \n",
       "11  34244  video/video_de-DE.txt  de_core_news_sm              0      german   \n",
       "1    8926  audio/audio_de-DE.txt  de_core_news_sm              0      german   \n",
       "6    1050  audio/audio_hi-IN.txt   en_core_web_sm              1       hindi   \n",
       "7    1240  audio/audio_pt-BR.txt  pt_core_news_sm              0  portuguese   \n",
       "15   3864  video/video_pt-BR.txt  pt_core_news_sm              0  portuguese   \n",
       "8    1250  audio/audio_ru-RU.txt  ru_core_news_sm              0     russian   \n",
       "16    135     video/video_ru.txt  ru_core_news_sm              0     russian   \n",
       "4    2690  audio/audio_es-MX.txt  es_core_news_sm              0     spanish   \n",
       "3     129  audio/audio_es-ES.txt  es_core_news_sm              0     spanish   \n",
       "17      3  video/video_sv-SE.txt    xx_sent_ud_sm              1     swedish   \n",
       "9    1585  audio/audio_sv-SE.txt    xx_sent_ud_sm              1     swedish   \n",
       "10   4277  audio/audio_tr-TR.txt   en_core_web_sm              1     turkish   \n",
       "\n",
       "            stopwords additionalInfo  \n",
       "19   stopwords-en.txt                 \n",
       "2    stopwords-en.txt                 \n",
       "14   stopwords-en.txt                 \n",
       "12   stopwords-en.txt                 \n",
       "18   stopwords-en.txt                 \n",
       "13   stopwords-fr.txt                 \n",
       "5    stopwords-fr.txt                 \n",
       "11   stopwords-de.txt                 \n",
       "1    stopwords-de.txt                 \n",
       "6   hi-stop-words.txt                 \n",
       "7    stopwords-pt.txt                 \n",
       "15   stopwords-pt.txt                 \n",
       "8    stopwords-ru.txt                 \n",
       "16   stopwords-ru.txt                 \n",
       "4    stopwords-es.txt                 \n",
       "3    stopwords-es.txt                 \n",
       "17  sv-stop-words.txt                 \n",
       "9   sv-stop-words.txt                 \n",
       "10  tr-stop-words.txt                 "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_transcripts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concatenate the audio and video transcripts for each language"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "f1 = pd.read_csv('../data/all-transcripts/audio/audio_fr-FR.txt',header=None,names=['text'])\n",
    "f2 = pd.read_csv('../data/all-transcripts/video/video_fr-FR.txt',header=None,names=['text'])\n",
    "french = pd.concat([f1,f2])\n",
    "\n",
    "g1 = pd.read_csv('../data/all-transcripts/audio/audio_de-DE.txt',header=None,names=['text'])\n",
    "g2 = pd.read_csv('../data/all-transcripts/video/video_de-DE.txt',header=None,names=['text'])\n",
    "german = pd.concat([g1,g2])\n",
    "\n",
    "hindi = pd.read_csv('../data/all-transcripts/audio/audio_hi-IN.txt',header=None,names=['text'])\n",
    "\n",
    "p1 = pd.read_csv('../data/all-transcripts/audio/audio_pt-BR.txt',header=None,names=['text'])\n",
    "p2 = pd.read_csv('../data/all-transcripts/video/video_pt-BR.txt',header=None,names=['text'])\n",
    "portuguese = pd.concat([p1,p2])\n",
    "\n",
    "r1 = pd.read_csv('../data/all-transcripts/audio/audio_ru-RU.txt',header=None,names=['text'])\n",
    "r2 = pd.read_csv('../data/all-transcripts/video/video_ru.txt',header=None,names=['text'])\n",
    "russian = pd.concat([r1,r2])\n",
    "\n",
    "s1 = pd.read_csv('../data/all-transcripts/audio/audio_es-ES.txt',header=None,names=['text'])\n",
    "s2 = pd.read_csv('../data/all-transcripts/audio/audio_es-MX.txt',header=None,names=['text'])\n",
    "spanish = pd.concat([s1,s2])\n",
    "\n",
    "w1 = pd.read_csv('../data/all-transcripts/audio/audio_sv-SE.txt',header=None,names=['text'])\n",
    "w2 = pd.read_csv('../data/all-transcripts/video/video_sv-SE.txt',header=None,names=['text'])\n",
    "swedish = pd.concat([w1,w2])\n",
    "\n",
    "turkish = pd.read_csv('../data/all-transcripts/audio/audio_tr-TR.txt',header=None,names=['text'])\n",
    "\n",
    "c1 = pd.read_csv('../data/all-transcripts/video/video_zh-CN.txt',header=None,names=['text'])\n",
    "c2 = pd.read_csv('../data/all-transcripts/video/video_zh-HK.txt',header=None,names=['text'])\n",
    "chinese = pd.concat([c1,c2])"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "french.to_csv('../data/all-transcripts/transcripts-all-fr.csv',index=None,header=None)\n",
    "german.to_csv('../data/all-transcripts/transcripts-all-de.csv',index=None,header=None)\n",
    "hindi.to_csv('../data/all-transcripts/transcripts-all-hi.csv',index=None,header=None)\n",
    "portuguese.to_csv('../data/all-transcripts/transcripts-all-pt.csv',index=None,header=None)\n",
    "russian.to_csv('../data/all-transcripts/transcripts-all-ru.csv',index=None,header=None)\n",
    "spanish.to_csv('../data/all-transcripts/transcripts-all-es.csv',index=None,header=None)\n",
    "swedish.to_csv('../data/all-transcripts/transcripts-all-sv.csv',index=None,header=None)\n",
    "turkish.to_csv('../data/all-transcripts/transcripts-all-tr.csv',index=None,header=None)\n",
    "chinese.to_csv('../data/all-transcripts/transcripts-all-zh.csv',index=None,header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "french = pd.read_csv('../data/all-transcripts/transcripts-all-fr.csv',names=['text'])\n",
    "german = pd.read_csv('../data/all-transcripts/transcripts-all-de.csv',names=['text'])\n",
    "hindi = pd.read_csv('../data/all-transcripts/transcripts-all-hi.csv',names=['text'])\n",
    "portuguese = pd.read_csv('../data/all-transcripts/transcripts-all-pt.csv',names=['text'])\n",
    "russian = pd.read_csv('../data/all-transcripts/transcripts-all-ru.csv',names=['text'])\n",
    "spanish = pd.read_csv('../data/all-transcripts/transcripts-all-es.csv',names=['text'])\n",
    "swedish = pd.read_csv('../data/all-transcripts/transcripts-all-sv.csv',names=['text'])\n",
    "turkish = pd.read_csv('../data/all-transcripts/transcripts-all-tr.csv',names=['text'])\n",
    "# chinese = pd.read_csv('../data/all-transcripts/transcripts-zh-pretokenized.csv',names=['text'])\n",
    "# arabic = pd.read_csv('../data/all-transcripts/transcripts-ar-pretokenized.csv',names=['text'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create an iterable for generating the ngrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "langs = [(french,'fr','fr_core_news_sm',None),\n",
    "         (german,'de','de_core_news_sm',None),\n",
    "         (hindi,'hi','en_core_web_sm','hi-stop-words.txt'),\n",
    "         (portuguese,'pt','pt_core_news_sm',None),\n",
    "         (russian,'ru','ru_core_news_sm',None),\n",
    "         (spanish,'es','es_core_news_sm',None),\n",
    "         (swedish,'sv','xx_sent_ud_sm','sv-stop-words.txt'),\n",
    "         (turkish,'tr','en_core_web_sm','tr-stop-words.txt')\n",
    "         ]\n",
    "# (chinese,'zh','zh_core_web_sm','stopwords-zh.txt'),\n",
    "# (arabic,'ar','None','ar-stop-words.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "!head -n 10 ../data/transcripts/transcripts-all-hi.csv > ../data/transcripts/sample.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Counting ngrams: 100%|██████████| 1/1 [00:06<00:00,  6.73s/it]\n",
      "Calculating TFIDF: 100%|██████████| 3433/3433 [00:04<00:00, 849.00it/s]\n",
      "Calculating PMI: 100%|██████████| 8612/8612 [00:10<00:00, 840.57it/s] \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('Ok Google', 9.154510487015513),\n",
       " ('facilite épreuve', 9.154510487015513),\n",
       " ('habiller conséquence', 9.154510487015513),\n",
       " ('conséquence tendances', 9.154510487015513),\n",
       " ('attaquer compotes', 9.154510487015513),\n",
       " ('COS pi', 9.154510487015513),\n",
       " ('pi rationalisme', 9.154510487015513),\n",
       " ('rationalisme corollaire', 9.154510487015513),\n",
       " ('commission chargée', 9.154510487015513),\n",
       " ('pape organise', 9.154510487015513),\n",
       " ('bravo Bravo', 9.154510487015513),\n",
       " ('Bravo multiplicité', 9.154510487015513),\n",
       " ('horizon confondu', 9.154510487015513),\n",
       " ('confondu témoignant', 9.154510487015513),\n",
       " ('Douleur aveux', 9.154510487015513)]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sanity check\n",
    "import vocabulary_utils as vu\n",
    "importlib.reload(vu)\n",
    "ua = pd.read_csv('../data/all-transcripts/sample-fr.csv',header=None,names=['text'])\n",
    "combined_list,_ = vu.getNgramsSpacy(ua['text'],spacy_model='fr_core_news_sm',stopwords=None,max_n=15,LLMvocab=xlm_roberta_vocab,pmi=True)\n",
    "combined_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Counting ngrams: 100%|██████████| 1/1 [00:06<00:00,  6.28s/it]\n",
      "Calculating TFIDF: 100%|██████████| 3433/3433 [00:04<00:00, 839.64it/s] \n",
      "Calculating PMI: 100%|██████████| 8612/8612 [00:09<00:00, 893.44it/s] \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('Ok Google', 9.154510487015513),\n",
       " ('facilite épreuve', 9.154510487015513),\n",
       " ('habiller conséquence', 9.154510487015513),\n",
       " ('conséquence tendances', 9.154510487015513),\n",
       " ('attaquer compotes', 9.154510487015513),\n",
       " ('COS pi', 9.154510487015513),\n",
       " ('pi rationalisme', 9.154510487015513),\n",
       " ('rationalisme corollaire', 9.154510487015513),\n",
       " ('commission chargée', 9.154510487015513),\n",
       " ('pape organise', 9.154510487015513),\n",
       " ('bravo Bravo', 9.154510487015513),\n",
       " ('Bravo multiplicité', 9.154510487015513),\n",
       " ('horizon confondu', 9.154510487015513),\n",
       " ('confondu témoignant', 9.154510487015513),\n",
       " ('Douleur aveux', 9.154510487015513),\n",
       " ('descend singe', 9.154510487015513),\n",
       " ('Yapi relation', 9.154510487015513),\n",
       " ('relation Écritures', 9.154510487015513),\n",
       " ('sache déclarer', 9.154510487015513),\n",
       " ('embrigadement Toulouse', 9.154510487015513),\n",
       " ('valeur reperdre', 9.154510487015513),\n",
       " ('équivalent congé', 9.154510487015513),\n",
       " ('menteur mytho', 9.154510487015513),\n",
       " ('enjeu lampes', 9.154510487015513),\n",
       " ('risques prête', 9.154510487015513),\n",
       " ('régionaux départementaux', 9.154510487015513),\n",
       " ('soucie médianes', 9.154510487015513),\n",
       " ('médianes trompes', 9.154510487015513),\n",
       " ('trompes rage', 9.154510487015513),\n",
       " ('rage tromperai', 9.154510487015513),\n",
       " ('Oublier crédit', 9.154510487015513),\n",
       " ('chansons New', 9.154510487015513),\n",
       " ('New BD', 9.154510487015513),\n",
       " ('gloire pend', 9.154510487015513),\n",
       " ('pend dresser', 9.154510487015513),\n",
       " ('Paso amendes', 9.154510487015513),\n",
       " ('planète lune', 9.154510487015513),\n",
       " ('négation loyauté', 9.154510487015513),\n",
       " ('crétins lâches', 9.154510487015513),\n",
       " ('lâches hypocrites', 9.154510487015513),\n",
       " ('englobe combattons', 9.154510487015513),\n",
       " ('combattons transhumanisme', 9.154510487015513),\n",
       " ('Blue Beam', 9.154510487015513),\n",
       " ('grand-chose structuration', 9.154510487015513),\n",
       " ('civilisations cultures', 9.154510487015513),\n",
       " ('compote East', 9.154510487015513),\n",
       " ('5e édition', 9.154510487015513),\n",
       " ('édition distribuer', 9.154510487015513),\n",
       " ('qualifié domaines', 9.154510487015513),\n",
       " ('mensonges électrique', 9.154510487015513),\n",
       " ('restreindre droits', 9.154510487015513),\n",
       " ('droits fondamentaux', 9.154510487015513),\n",
       " ('fondamentaux esprits', 9.154510487015513),\n",
       " ('recettes intégration', 9.154510487015513),\n",
       " ('converge populations', 9.154510487015513),\n",
       " ('TP nourriture', 9.154510487015513),\n",
       " ('commandements limitation', 9.154510487015513),\n",
       " ('limitation Humanité', 9.154510487015513),\n",
       " ('Humanité 500000000', 9.154510487015513),\n",
       " ('500000000 dépôt', 9.154510487015513),\n",
       " ('dépôt Mondial', 9.154510487015513),\n",
       " ('Droguer tapé', 9.154510487015513),\n",
       " ('Claire Séverac', 9.154510487015513),\n",
       " ('bec crucifiés', 9.154510487015513),\n",
       " ('parfait fleurs', 9.154510487015513),\n",
       " ('réellement tenants', 9.154510487015513),\n",
       " ('tenants aboutissants', 9.154510487015513),\n",
       " ('Pardon calculer', 9.154510487015513),\n",
       " ('versé 31', 9.154510487015513),\n",
       " ('cédé dictature', 9.154510487015513),\n",
       " ('dictature génocide', 9.154510487015513),\n",
       " ('initiatives culottée', 9.154510487015513),\n",
       " ('affole cherche', 9.154510487015513),\n",
       " ('cherche enfermer', 9.154510487015513),\n",
       " ('communauté combattant', 9.154510487015513),\n",
       " ('Passer moments', 9.154510487015513),\n",
       " ('frères soeur', 9.154510487015513),\n",
       " ('primordiale prioritaire', 9.154510487015513),\n",
       " ('voyais respire', 9.154510487015513),\n",
       " ('sirop résultat', 9.154510487015513),\n",
       " ('aucun protocole', 9.154510487015513),\n",
       " ('hausse impôts', 9.154510487015513),\n",
       " ('listing listes', 9.154510487015513),\n",
       " ('identifié cataloguée', 9.154510487015513),\n",
       " ('Australie pleine', 9.154510487015513),\n",
       " ('24 États', 9.154510487015513),\n",
       " ('États américains', 9.154510487015513),\n",
       " ('américains oppose', 9.154510487015513),\n",
       " ('refuses obligation', 9.154510487015513),\n",
       " ('uniquement telegram', 9.154510487015513),\n",
       " ('aimer mesure', 9.154510487015513),\n",
       " ('mesure soutenir', 9.154510487015513),\n",
       " ('tricheurs correcteur', 9.154510487015513),\n",
       " ('correcteur complot', 9.154510487015513),\n",
       " ('mein stream', 9.154510487015513),\n",
       " ('Momo Diallo', 9.154510487015513),\n",
       " ('adolescente 14', 9.154510487015513),\n",
       " ('démissionne mandat', 9.154510487015513),\n",
       " ('israélienne Histoire', 9.154510487015513),\n",
       " ('Histoire Indiens', 9.154510487015513),\n",
       " ('Indiens blessé', 9.154510487015513),\n",
       " ('Marvin tong', 9.154510487015513),\n",
       " ('tong star', 9.154510487015513),\n",
       " ('star Tic', 9.154510487015513),\n",
       " ('Tic Toc', 9.154510487015513),\n",
       " ('1,9 followers', 9.154510487015513),\n",
       " ('répertoriées vachement', 9.154510487015513),\n",
       " ('NHS confirme', 9.154510487015513),\n",
       " ('effectivement Sun', 9.154510487015513),\n",
       " ('TCD probablement', 9.154510487015513),\n",
       " ('probablement lié', 9.154510487015513),\n",
       " ('Nouvelle Calédonie', 9.154510487015513),\n",
       " ('domicile établissement', 9.154510487015513),\n",
       " ('établissement social', 9.154510487015513),\n",
       " ('présumé décéder', 9.154510487015513),\n",
       " ('décéder covid-19', 9.154510487015513),\n",
       " ('covid-19 certificat', 9.154510487015513),\n",
       " ('atteste conséquent', 9.154510487015513),\n",
       " ('conséquent défunts', 9.154510487015513),\n",
       " ('potentiellement contagieux', 9.154510487015513),\n",
       " ('toilette mortuaire', 9.154510487015513),\n",
       " ('conservation interdits', 9.154510487015513),\n",
       " ('interdits corps', 9.154510487015513),\n",
       " ('zelenko kik', 9.154510487015513),\n",
       " ('statistiques voudra', 9.154510487015513),\n",
       " ('Haute Autorité', 9.154510487015513),\n",
       " ('dramatique avenir', 9.154510487015513),\n",
       " ('démissionner hommage', 9.154510487015513),\n",
       " ('médicament fizer', 9.154510487015513),\n",
       " ('1550 252551', 9.154510487015513),\n",
       " ('pronostic vital', 9.154510487015513),\n",
       " ('détaillé Freezer', 9.154510487015513),\n",
       " ('RS planning', 9.154510487015513),\n",
       " ('CHU Montpellier', 9.154510487015513),\n",
       " ('directrice Académie', 9.154510487015513),\n",
       " ('Académie Organisation', 9.154510487015513),\n",
       " ('Lyon Gerland', 9.154510487015513),\n",
       " ('JP JPO', 9.154510487015513),\n",
       " ('république défend', 9.154510487015513),\n",
       " ('souligner conflits', 9.154510487015513),\n",
       " ('conflits intérêts', 9.154510487015513),\n",
       " ('consommateur patient', 9.154510487015513),\n",
       " ('patient consommateurs', 9.154510487015513),\n",
       " ('consommateurs patientes', 9.154510487015513),\n",
       " ('endroit députée', 9.154510487015513),\n",
       " ('Claude Évin', 9.154510487015513),\n",
       " ('Fabre Henri', 9.154510487015513),\n",
       " ('Henri Nallet', 9.154510487015513),\n",
       " ('Envoyer Élisabeth', 9.154510487015513),\n",
       " ('Élisabeth Hubert', 9.154510487015513),\n",
       " ('infirmières étonné', 9.154510487015513),\n",
       " ('352 Valérie', 9.154510487015513),\n",
       " ('Étudiant 1941', 9.154510487015513),\n",
       " ('centre médico', 9.154510487015513),\n",
       " ('médico psychologue', 9.154510487015513),\n",
       " ('ecoeurant quitte', 9.154510487015513),\n",
       " ('témoin assisté', 9.154510487015513),\n",
       " ('présidentielles jure', 9.154510487015513),\n",
       " ('jure voterai', 9.154510487015513),\n",
       " ('ennemi pedophilie', 9.154510487015513),\n",
       " ('standing ovation', 9.154510487015513),\n",
       " ('rétablis guillotine', 9.154510487015513),\n",
       " ('guillotine deviendrai', 9.154510487015513),\n",
       " ('deviendrai Robespierre', 9.154510487015513),\n",
       " ('Robespierre pitié', 9.154510487015513),\n",
       " ('pitié viole', 9.154510487015513),\n",
       " ('fiers rigole', 9.154510487015513),\n",
       " ('heureuses heureux', 9.154510487015513),\n",
       " ('attrape termine', 9.154510487015513),\n",
       " ('autoriser samedia', 9.154510487015513),\n",
       " ('Alain Delon', 9.154510487015513),\n",
       " ('meilleure robe', 9.154510487015513),\n",
       " ('robe attache', 9.154510487015513),\n",
       " ('attache souffrir', 9.154510487015513),\n",
       " ('souffrir propres', 9.154510487015513),\n",
       " ('attachés détache', 9.154510487015513),\n",
       " ('WC rattachés', 9.154510487015513),\n",
       " ('torture écris', 9.154510487015513),\n",
       " ('écris pleure', 9.154510487015513),\n",
       " ('citer buvait', 9.154510487015513),\n",
       " ('croyais Marbella', 9.154510487015513),\n",
       " ('jouvence vieillis', 9.154510487015513),\n",
       " ('Mel Gibson', 9.154510487015513),\n",
       " ('Gibson connait', 9.154510487015513),\n",
       " ('connait buveur', 9.154510487015513),\n",
       " ('bonhomme cavale', 9.154510487015513),\n",
       " ('cavale Thaïlande', 9.154510487015513),\n",
       " ('maisons closes', 9.154510487015513),\n",
       " ('closes riches', 9.154510487015513),\n",
       " ('vieux séculaire', 9.154510487015513),\n",
       " ('subordonnées troufion', 9.154510487015513),\n",
       " ('troufion entrés', 9.154510487015513),\n",
       " ('entrés brèche', 9.154510487015513),\n",
       " ('craquer fillette', 9.154510487015513),\n",
       " ('maçonnique rite', 9.154510487015513),\n",
       " ('pièces détachées', 9.154510487015513),\n",
       " ('proximité DVD', 9.154510487015513),\n",
       " ('DVD proxénétisme', 9.154510487015513),\n",
       " ('classique club', 9.154510487015513),\n",
       " ('mythomane langues', 9.154510487015513),\n",
       " ('langues délient', 9.154510487015513),\n",
       " ('gogo Pedro', 9.154510487015513),\n",
       " ('Berger taille', 9.154510487015513),\n",
       " ('décerne honneurs', 9.154510487015513),\n",
       " ('horreurs particulier', 9.154510487015513),\n",
       " ('dessiné robes', 9.154510487015513),\n",
       " ('tablier rose', 9.154510487015513),\n",
       " ('rose marchait', 9.154510487015513),\n",
       " ('foutait existais', 9.154510487015513),\n",
       " ('obliger chier', 9.154510487015513),\n",
       " ('mettait récipients', 9.154510487015513),\n",
       " ('Légion honneur', 9.154510487015513),\n",
       " ('restes Ouais', 9.154510487015513),\n",
       " ('fournisseur stars', 9.154510487015513),\n",
       " ('voyous Retrouverez', 9.154510487015513),\n",
       " ('Girard Forrest', 9.154510487015513),\n",
       " ('Serge Dassault', 9.154510487015513),\n",
       " ('poussé cris', 9.154510487015513),\n",
       " ('vider diabète', 9.154510487015513),\n",
       " ('diabète diarrhée', 9.154510487015513),\n",
       " ('ligne cote', 9.154510487015513),\n",
       " ('remplisse rajoutes', 9.154510487015513),\n",
       " ('rajoutes laxative', 9.154510487015513),\n",
       " ('iras fournir', 9.154510487015513),\n",
       " ('faisais évolue', 9.154510487015513),\n",
       " ('limousine Mercedes', 9.154510487015513),\n",
       " ('instructions dévoiler', 9.154510487015513),\n",
       " ('41 fièvre', 9.154510487015513),\n",
       " ('Saoudiens qatari', 9.154510487015513),\n",
       " ('couper morceaux', 9.154510487015513),\n",
       " ('max maltais', 9.154510487015513),\n",
       " ('doublé peindre', 9.154510487015513),\n",
       " ('peindre 88', 9.154510487015513),\n",
       " ('normalement dû', 9.154510487015513),\n",
       " ('apprécie marathon', 9.154510487015513),\n",
       " ('Gilles Halim', 9.154510487015513),\n",
       " ('Halim moral', 9.154510487015513),\n",
       " ('moral Guillou', 9.154510487015513),\n",
       " ('violet MAAF', 9.154510487015513),\n",
       " ('86 juré', 9.154510487015513),\n",
       " ('tours viols', 9.154510487015513),\n",
       " ('putin connard', 9.154510487015513),\n",
       " ('concubine reviens', 9.154510487015513),\n",
       " ('perron crèche', 9.154510487015513),\n",
       " ('risque énerver', 9.154510487015513),\n",
       " ('North Africa', 9.154510487015513),\n",
       " ('bain scandale', 9.154510487015513),\n",
       " ('scandale diplomatique', 9.154510487015513),\n",
       " ('diplomatique sac', 9.154510487015513),\n",
       " ('Hassan II', 9.154510487015513),\n",
       " ('II juif', 9.154510487015513),\n",
       " ('descendant Prophète', 9.154510487015513),\n",
       " ('salopes utilisant', 9.154510487015513),\n",
       " ('sinueuses entrer', 9.154510487015513),\n",
       " ('entrer difficilement', 9.154510487015513),\n",
       " ('couru lapins', 9.154510487015513),\n",
       " ('lapins détalé', 9.154510487015513),\n",
       " ('intervenus tire', 9.154510487015513),\n",
       " ('fusil chasse', 9.154510487015513),\n",
       " ('habites essayé', 9.154510487015513),\n",
       " ('copain gitans', 9.154510487015513),\n",
       " ('bluetooth Bella', 9.154510487015513),\n",
       " ('Bella Hadid', 9.154510487015513),\n",
       " ('tombés civil', 9.154510487015513),\n",
       " ('civil béquilles', 9.154510487015513),\n",
       " ('béquilles balcon', 9.154510487015513),\n",
       " ('balcon fouillé', 9.154510487015513),\n",
       " ('embarqué regardé', 9.154510487015513),\n",
       " ('caïd samedi', 9.154510487015513),\n",
       " ('surpris teneur', 9.154510487015513),\n",
       " ('teneur copine', 9.154510487015513),\n",
       " ('exprès Avesnes-sur-Helpe', 9.154510487015513),\n",
       " ('connaissait Dupont', 9.154510487015513),\n",
       " ('consommation personnelle', 9.154510487015513),\n",
       " ('rencontré Fleury', 9.154510487015513),\n",
       " ('Fleury Mérogis', 9.154510487015513),\n",
       " ('moquer signé', 9.154510487015513),\n",
       " ('cynisme effroyable', 9.154510487015513),\n",
       " ('effroyable frère', 9.154510487015513),\n",
       " ('personnellement mimiques', 9.154510487015513),\n",
       " ('traîner effets', 9.154510487015513),\n",
       " ('prenait héroïne', 9.154510487015513),\n",
       " ('héroïne passez', 9.154510487015513),\n",
       " ('passez gratter', 9.154510487015513),\n",
       " ('gratter avant-bras', 9.154510487015513),\n",
       " ('science fiction', 9.154510487015513),\n",
       " ('vent sataniste', 9.154510487015513),\n",
       " ('saches Élise', 9.154510487015513),\n",
       " ('Élise Lucet', 9.154510487015513),\n",
       " ('secte Moon', 9.154510487015513),\n",
       " ('ventre dégueulasses', 9.154510487015513),\n",
       " ('balle Kabbale', 9.154510487015513),\n",
       " ('Kabbale croyances', 9.154510487015513),\n",
       " ('exponentielle Dreno', 9.154510487015513),\n",
       " ('bourrés dosette', 9.154510487015513),\n",
       " ('revendre Saudia', 9.154510487015513),\n",
       " ('avion de-ci', 9.154510487015513),\n",
       " ('de-ci de-là', 9.154510487015513),\n",
       " ('organisée opposition', 9.154510487015513),\n",
       " ('désolidarise propos', 9.154510487015513),\n",
       " ('Lyme hôpitaux', 9.154510487015513),\n",
       " ('côte recherche', 9.154510487015513),\n",
       " ('vendus 55', 9.154510487015513),\n",
       " ('Canada Effectivement', 9.154510487015513),\n",
       " ('guéri lait', 9.154510487015513),\n",
       " ('lait recours', 9.154510487015513),\n",
       " ('recours systématique', 9.154510487015513),\n",
       " ('taux glycémie', 9.154510487015513),\n",
       " ('glycémie favoriser', 9.154510487015513),\n",
       " ('favoriser développement', 9.154510487015513),\n",
       " ('indigne couleur', 9.154510487015513),\n",
       " ('couleur servait', 9.154510487015513),\n",
       " ('raid M6', 9.154510487015513),\n",
       " ('M6 vire', 9.154510487015513),\n",
       " ('substance dangereuse', 9.154510487015513),\n",
       " ('lisation déplace', 9.154510487015513),\n",
       " ('déplace maquillée', 9.154510487015513),\n",
       " ('composition thé', 9.154510487015513),\n",
       " ('dessin animé', 9.154510487015513),\n",
       " ('masque ouvre', 9.154510487015513),\n",
       " ('tousse pagaille', 9.154510487015513),\n",
       " ('domination voulant', 9.154510487015513),\n",
       " ('paysage karting', 9.154510487015513),\n",
       " ('sauver notaire', 9.154510487015513),\n",
       " ('dégage rats', 9.154510487015513),\n",
       " ('fragiles faible', 9.154510487015513),\n",
       " ('enverrai tarte', 9.154510487015513),\n",
       " ('Djibril défonce', 9.154510487015513),\n",
       " ('Annick hymne', 9.154510487015513),\n",
       " ('mondes engloutis', 9.154510487015513),\n",
       " ('ramène Lucie', 9.154510487015513),\n",
       " ('Locon existent', 9.154510487015513),\n",
       " ('religions bénédiction', 9.154510487015513),\n",
       " ('intégré discours', 9.154510487015513),\n",
       " ('contemporaine 33000', 9.154510487015513),\n",
       " ('33000 infirmier', 9.154510487015513),\n",
       " ('grille ajoute', 9.154510487015513),\n",
       " ('renvoies narratif', 9.154510487015513),\n",
       " ('narratif sonorité', 9.154510487015513),\n",
       " ('relancé 1429', 9.154510487015513),\n",
       " ('Roselyne Bachelot', 9.154510487015513),\n",
       " ('limiter cuisine', 9.154510487015513),\n",
       " ('cuisine hauteur', 9.154510487015513),\n",
       " ('hauteur 94', 9.154510487015513),\n",
       " ('tempérer optimisme', 9.154510487015513),\n",
       " ('optimisme citant', 9.154510487015513),\n",
       " ('théorie actuellement', 9.154510487015513),\n",
       " ('actuellement progression', 9.154510487015513),\n",
       " ('HIV 4000', 9.154510487015513),\n",
       " ('détourner millions', 9.154510487015513),\n",
       " ('inhabituel Lindsay', 9.154510487015513),\n",
       " ('artificiellement gonflé', 9.154510487015513),\n",
       " ('supérieur 82,5', 9.154510487015513),\n",
       " ('isation Cazarre', 9.154510487015513),\n",
       " ('nombres croisés', 9.154510487015513),\n",
       " ('croisés périodes', 9.154510487015513),\n",
       " ('périodes Creuse', 9.154510487015513),\n",
       " ('convoquer conseil', 9.154510487015513),\n",
       " ('scientifiquement prouvé', 9.154510487015513),\n",
       " ('prouvé immense', 9.154510487015513),\n",
       " ('réduit cendres', 9.154510487015513),\n",
       " ('poches pleines', 9.154510487015513),\n",
       " ('entraxe cytomégalovirus', 9.154510487015513),\n",
       " ('cytomégalovirus varicelle', 9.154510487015513),\n",
       " ('varicelle utiliser', 9.154510487015513),\n",
       " ('utiliser embryons', 9.154510487015513),\n",
       " ('handicapées mentales', 9.154510487015513),\n",
       " ('6000 avortements', 9.154510487015513),\n",
       " ('né péché', 9.154510487015513),\n",
       " ('péché collabore', 9.154510487015513),\n",
       " ('criminel déclare', 9.154510487015513),\n",
       " ('membre Shin', 9.154510487015513),\n",
       " ('Shin Beth', 9.154510487015513),\n",
       " ('Beth Yacov', 9.154510487015513),\n",
       " ('Yacov charrette', 9.154510487015513),\n",
       " ('sincèrement aient', 9.154510487015513),\n",
       " ('joue reportage', 9.154510487015513),\n",
       " ('pensent buté', 9.154510487015513),\n",
       " ('fervent Anti', 9.154510487015513),\n",
       " ('Anti Sioniste', 9.154510487015513),\n",
       " ('revenez leboncoin', 9.154510487015513),\n",
       " ('Ensuite occupera', 9.154510487015513),\n",
       " ('chrétienne islamique', 9.154510487015513),\n",
       " ('ghost town', 9.154510487015513),\n",
       " ('ripoux francs-maçons', 9.154510487015513),\n",
       " ('National station', 9.154510487015513),\n",
       " ('station Mossad', 9.154510487015513),\n",
       " ('Mossad territoire', 9.154510487015513),\n",
       " ('cher démarrer', 9.154510487015513),\n",
       " ('Ploncard Assac', 9.154510487015513),\n",
       " ('cercle audace', 9.154510487015513),\n",
       " ('audace récent', 9.154510487015513),\n",
       " ('raisons conclusions', 9.154510487015513),\n",
       " ('médiatiques attirer', 9.154510487015513),\n",
       " ('attirer marchandise', 9.154510487015513),\n",
       " ('quasiment terminé', 9.154510487015513),\n",
       " ('Choses amusantes', 9.154510487015513),\n",
       " ('fruit spermatozoïde', 9.154510487015513),\n",
       " ('spermatozoïde garée', 9.154510487015513),\n",
       " ('garée Roger', 9.154510487015513),\n",
       " ('Roger Auque', 9.154510487015513),\n",
       " ('Auque coureur', 9.154510487015513),\n",
       " ('coureur déterrer', 9.154510487015513),\n",
       " ('déterrer couronner', 9.154510487015513),\n",
       " ('dernièrement Éric', 9.154510487015513),\n",
       " ('université lyonnaise', 9.154510487015513),\n",
       " ('dernières déclarations', 9.154510487015513),\n",
       " ('fours crématoires', 9.154510487015513),\n",
       " ('salue 1638', 9.154510487015513),\n",
       " ('accompli autant', 9.154510487015513),\n",
       " ('moitié succès', 9.154510487015513),\n",
       " ('Bilbo fameuse', 9.154510487015513),\n",
       " ('reproductive diminuera', 9.154510487015513),\n",
       " ('responsable antipathique', 9.154510487015513),\n",
       " ('traversée désert', 9.154510487015513),\n",
       " ('applications énorme', 9.154510487015513),\n",
       " ('décoder impulsions', 9.154510487015513),\n",
       " ('impulsions électriques', 9.154510487015513),\n",
       " ('électriques émise', 9.154510487015513),\n",
       " ('smartphone tablette', 9.154510487015513),\n",
       " ('géants web', 9.154510487015513),\n",
       " ('web travaille', 9.154510487015513),\n",
       " ('minute partir', 9.154510487015513),\n",
       " ('patron Tesla', 9.154510487015513),\n",
       " ('développer type', 9.154510487015513),\n",
       " ('type électrodes', 9.154510487015513),\n",
       " ('Film cheveu', 9.154510487015513),\n",
       " ('implanter cortex', 9.154510487015513),\n",
       " ('boîte crânienne', 9.154510487015513),\n",
       " ('parfaitement cartographier', 9.154510487015513),\n",
       " ('dizaine drone', 9.154510487015513),\n",
       " ('opérations militaires', 9.154510487015513),\n",
       " ('open space', 9.154510487015513),\n",
       " ('space totalement', 9.154510487015513),\n",
       " ('totalement silencieux', 9.154510487015513),\n",
       " ('silencieux salarié', 9.154510487015513),\n",
       " ('salarié implant', 9.154510487015513),\n",
       " ('intentionnés pirater', 9.154510487015513),\n",
       " ('envies besoins', 9.154510487015513),\n",
       " ('104 min', 9.154510487015513),\n",
       " ('adorée imagine', 9.154510487015513),\n",
       " ('appareils mécanisme', 9.154510487015513),\n",
       " ('mécanisme robotisée', 9.154510487015513),\n",
       " ('montré surnoms', 9.154510487015513),\n",
       " ('Baudry nouvelles', 9.154510487015513),\n",
       " ('récente manifestation', 9.154510487015513),\n",
       " ('libération Nouvel', 9.154510487015513),\n",
       " ('Nouvel Obs', 9.154510487015513),\n",
       " ('présentateur présentait', 9.154510487015513),\n",
       " ('diffusent parties', 9.154510487015513),\n",
       " ('minuterie Cardo', 9.154510487015513),\n",
       " ('ARN messager', 9.154510487015513),\n",
       " ('messager reprogrammer', 9.154510487015513),\n",
       " ('reprogrammer ADN', 9.154510487015513),\n",
       " ('transformer injecteurs', 9.154510487015513),\n",
       " ('injecteurs potentiel', 9.154510487015513),\n",
       " ('potentiel démon', 9.154510487015513),\n",
       " ('démon contiennent', 9.154510487015513),\n",
       " ('contiennent proteinogene', 9.154510487015513),\n",
       " ('proteinogene magnéto', 9.154510487015513),\n",
       " ('mental holistique', 9.154510487015513),\n",
       " ('holistique biais', 9.154510487015513),\n",
       " ('biais 5G.', 9.154510487015513),\n",
       " ('rassurer fouilles', 9.154510487015513),\n",
       " ('tables 700', 9.154510487015513),\n",
       " ('aille démonte', 9.154510487015513),\n",
       " ('Réduis marque', 9.154510487015513),\n",
       " ('marque franc-maçon', 9.154510487015513),\n",
       " ('porno inventé', 9.154510487015513),\n",
       " ('étonne poupées', 9.154510487015513),\n",
       " ('poupées gonflables', 9.154510487015513),\n",
       " ('déploiement robots', 9.154510487015513),\n",
       " ('robots environnement', 9.154510487015513),\n",
       " ('environnement intime', 9.154510487015513),\n",
       " ('habille Prada', 9.154510487015513),\n",
       " ('suscité indignation', 9.154510487015513),\n",
       " ('annoncer ouverture', 9.154510487015513),\n",
       " ('analysé collaboration', 9.154510487015513),\n",
       " ('collaboration agence', 9.154510487015513),\n",
       " ('agence expertise', 9.154510487015513),\n",
       " ('Louis Aragon', 9.154510487015513),\n",
       " ('débute 21h02', 9.154510487015513),\n",
       " ('situation calme', 9.154510487015513),\n",
       " ('contraint poire', 9.154510487015513),\n",
       " ('mains écartées', 9.154510487015513),\n",
       " ('débat affirmations', 9.154510487015513),\n",
       " ('affirmations libérer', 9.154510487015513),\n",
       " ('matraque telescopique', 9.154510487015513),\n",
       " ('rôle centimètres', 9.154510487015513),\n",
       " ('centimètres bloc', 9.154510487015513),\n",
       " ('bloc pierre', 9.154510487015513),\n",
       " ('mages Walhain', 9.154510487015513),\n",
       " ('élan remettre', 9.154510487015513),\n",
       " ('camion cash', 9.154510487015513),\n",
       " ('crabes agent', 9.154510487015513),\n",
       " ('reconnaissant dangerosité', 9.154510487015513),\n",
       " ('-on maintien', 9.154510487015513),\n",
       " ('danger entourant', 9.154510487015513),\n",
       " ('rapproche penche', 9.154510487015513),\n",
       " ('Finalement déchaînement', 9.154510487015513),\n",
       " ('populaires jaunes', 9.154510487015513),\n",
       " ('fauteuil roulant', 9.154510487015513),\n",
       " ('levé Léonie', 9.154510487015513),\n",
       " ('Luke Perry', 9.154510487015513),\n",
       " ('pratiques 40', 9.154510487015513),\n",
       " ('bande complète', 9.154510487015513),\n",
       " ('entièrement flou', 9.154510487015513),\n",
       " ('distinguer mouvements', 9.154510487015513),\n",
       " ('détention provisoire', 9.154510487015513),\n",
       " ('octobre violence', 9.154510487015513),\n",
       " ('maître Arie', 9.154510487015513),\n",
       " ('Arie Alimi', 9.154510487015513),\n",
       " ('tentative meurtre', 9.154510487015513),\n",
       " ('souhaité répondre', 9.154510487015513),\n",
       " ('répondre questions', 9.154510487015513),\n",
       " ('milieu journalisme', 9.154510487015513),\n",
       " ('journalisme montres', 9.154510487015513),\n",
       " ('François Daniel', 9.154510487015513),\n",
       " ('Daniel Chantal', 9.154510487015513),\n",
       " ('sujets février', 9.154510487015513),\n",
       " ('accueil doutes', 9.154510487015513),\n",
       " ('maltraitance coupable', 9.154510487015513),\n",
       " ('summum Américains', 9.154510487015513),\n",
       " ('préventif man', 9.154510487015513),\n",
       " ('nourrissons enlevés', 9.154510487015513),\n",
       " ('remonter tribunaux', 9.154510487015513),\n",
       " ('tribunaux commerce', 9.154510487015513),\n",
       " ('commerce Champions', 9.154510487015513),\n",
       " ('Kbis crimes', 9.154510487015513),\n",
       " ('photos vidéos', 9.154510487015513),\n",
       " ('informé existence', 9.154510487015513),\n",
       " ('souhaitons action', 9.154510487015513),\n",
       " ('angoisses nuits', 9.154510487015513),\n",
       " ('nuits blanches', 9.154510487015513),\n",
       " ('choc traumatisme', 9.154510487015513),\n",
       " ('mérite carte', 9.154510487015513),\n",
       " ('carte grise', 9.154510487015513),\n",
       " ('grise saigne', 9.154510487015513),\n",
       " ('procédés volonté', 9.154510487015513),\n",
       " ('bourreau SONEDE', 9.154510487015513),\n",
       " ('satellite alisiers', 9.154510487015513),\n",
       " ('sécu facturé', 9.154510487015513),\n",
       " ('facturé moyenne', 9.154510487015513),\n",
       " ('moyenne 7000', 9.154510487015513),\n",
       " ('strict minimum', 9.154510487015513),\n",
       " ('minimum Real', 9.154510487015513),\n",
       " ('Real Madrid', 9.154510487015513),\n",
       " ('Sandrine vol', 9.154510487015513),\n",
       " ('comptant kriminalis', 9.154510487015513),\n",
       " ('familles vivent', 9.154510487015513),\n",
       " ('tués créé', 9.154510487015513),\n",
       " ('1800 esclaves', 9.154510487015513),\n",
       " ('2800 esclave', 9.154510487015513),\n",
       " ('niçois considérés', 9.154510487015513),\n",
       " ('considérés antisémites', 9.154510487015513),\n",
       " ('Notre-Dame 2007', 9.154510487015513),\n",
       " ('2007 dirigeant', 9.154510487015513),\n",
       " ('Georges Floyd', 9.154510487015513),\n",
       " ('Regarder counter', 9.154510487015513),\n",
       " ('counter Ébola', 9.154510487015513),\n",
       " ('maîtrise techniques', 9.154510487015513),\n",
       " ('établis définitif', 9.154510487015513),\n",
       " ('définitif inscrit', 9.154510487015513),\n",
       " ('Nice boussole', 9.154510487015513),\n",
       " ('caché déconstruit', 9.154510487015513),\n",
       " ('connaissances organigramme', 9.154510487015513),\n",
       " ('battre cesse', 9.154510487015513),\n",
       " ('matrice arrivés', 9.154510487015513),\n",
       " ('Frontalement casse', 9.154510487015513),\n",
       " ('casse décor', 9.154510487015513),\n",
       " ('intégrité honnêteté', 9.154510487015513),\n",
       " ('honnêteté sincérité', 9.154510487015513),\n",
       " ('réelle clairvoyance', 9.154510487015513),\n",
       " ('maintient danser', 9.154510487015513),\n",
       " ('danser Jol', 9.154510487015513),\n",
       " ('Jol cachot', 9.154510487015513),\n",
       " ('émotion apprise', 9.154510487015513),\n",
       " ('apprise vain', 9.154510487015513),\n",
       " ('Marche eSPoir', 9.154510487015513),\n",
       " ('Leroy passion', 9.154510487015513),\n",
       " ('compotes montée', 8.461363306455567),\n",
       " ('montée COS', 8.461363306455567),\n",
       " ('lance commission', 8.461363306455567),\n",
       " ('chargée lutter', 8.461363306455567),\n",
       " ('bière réunis', 8.461363306455567),\n",
       " ('excellent bilan', 8.461363306455567),\n",
       " ('multiplicité intervenants', 8.461363306455567),\n",
       " ('intervenants diversité', 8.461363306455567),\n",
       " ('diversité intervenants', 8.461363306455567),\n",
       " ('matérialiste Saint-Esprit', 8.461363306455567),\n",
       " ('humanisme essentiel', 8.461363306455567),\n",
       " ('comprend anges', 8.461363306455567),\n",
       " ('prête assumer', 8.461363306455567),\n",
       " ('décliner territoires', 8.461363306455567),\n",
       " ('territoires régionaux', 8.461363306455567),\n",
       " ('milliards dollars', 8.461363306455567),\n",
       " ('tocards planète', 8.461363306455567),\n",
       " ('faisceaux air', 8.461363306455567),\n",
       " ('puissance totale', 8.461363306455567),\n",
       " ('Goliath minorité', 8.461363306455567),\n",
       " ('majorité puissants', 8.461363306455567),\n",
       " ('emblème décervelé', 8.461363306455567),\n",
       " ('décervelé compote', 8.461363306455567),\n",
       " ('East complotiste', 8.461363306455567),\n",
       " ('complotiste assume', 8.461363306455567),\n",
       " ('assume appuie', 8.461363306455567),\n",
       " ('factuel compréhension', 8.461363306455567),\n",
       " ('traces 5e', 8.461363306455567),\n",
       " ('traversé tract', 8.461363306455567),\n",
       " (\"tract puisqu'\", 8.461363306455567),\n",
       " ('éliminer comptent', 8.461363306455567),\n",
       " ('détermination affirmer', 8.461363306455567),\n",
       " ('efforts citoyens', 8.461363306455567),\n",
       " ('médecin chercheur', 8.461363306455567),\n",
       " ('chercheur chercheuse', 8.461363306455567),\n",
       " ('civile mensonges', 8.461363306455567),\n",
       " ('plan convient', 8.461363306455567),\n",
       " ('documents démontre', 8.461363306455567),\n",
       " ('manœuvre prétexte', 8.461363306455567),\n",
       " ('prétexte restreindre', 8.461363306455567),\n",
       " ('prêts ancêtres', 8.461363306455567),\n",
       " ('véritable avance', 8.461363306455567),\n",
       " ('croit vivre', 8.461363306455567),\n",
       " ('conclusion rentrent', 8.461363306455567),\n",
       " ('rentrent tenue', 8.461363306455567),\n",
       " ('CFA2 casser', 8.461363306455567),\n",
       " ('pensées voler', 8.461363306455567),\n",
       " ('voler recettes', 8.461363306455567),\n",
       " ('sommet État', 8.461363306455567),\n",
       " ('Séverac inversé', 8.461363306455567),\n",
       " ('inversé propose', 8.461363306455567),\n",
       " ('moutons traitées', 8.461363306455567),\n",
       " ('décerné prêts', 8.461363306455567),\n",
       " ('prêts bec', 8.461363306455567),\n",
       " ('fleurs bonbons', 8.461363306455567),\n",
       " ('procéder réellement', 8.461363306455567),\n",
       " ('aboutissants considérer', 8.461363306455567),\n",
       " ('considérer négationnistes', 8.461363306455567),\n",
       " ('empêcher réfléchir', 8.461363306455567),\n",
       " ('apporte contribution', 8.461363306455567),\n",
       " ('31 mars', 8.461363306455567),\n",
       " ('remercier cédé', 8.461363306455567),\n",
       " ('suisses plaint', 8.461363306455567),\n",
       " ('plaint encouragement', 8.461363306455567),\n",
       " ('Nathan simplement', 8.461363306455567),\n",
       " ('renoncer Mans', 8.461363306455567),\n",
       " ('savais rond', 8.461363306455567),\n",
       " ('dorment frères', 8.461363306455567),\n",
       " ('prisonniers voyait', 8.461363306455567),\n",
       " ('foule voyait', 8.461363306455567),\n",
       " ('fier sirop', 8.461363306455567),\n",
       " ('résultat demander', 8.461363306455567),\n",
       " ('endoctrinement lutter', 8.461363306455567),\n",
       " ('guérit prouve', 8.461363306455567),\n",
       " ('prouve plante', 8.461363306455567),\n",
       " ('plante demi', 8.461363306455567),\n",
       " ('demi voulu', 8.461363306455567),\n",
       " ('regardons faisons', 8.461363306455567),\n",
       " ('outils dominant', 8.461363306455567),\n",
       " ('dominant outils', 8.461363306455567),\n",
       " ('outils puissant', 8.461363306455567),\n",
       " ('fonctionne regardez', 8.461363306455567),\n",
       " ('manipule opinion', 8.461363306455567),\n",
       " ('dette covid', 8.461363306455567),\n",
       " ('Steve long', 8.461363306455567),\n",
       " ('général Australie', 8.461363306455567),\n",
       " ('refus mesures', 8.461363306455567),\n",
       " ('obligation Maxime', 8.461363306455567),\n",
       " ('Maxime Suède', 8.461363306455567),\n",
       " ('Suède passant', 8.461363306455567),\n",
       " ('passant masse', 8.461363306455567),\n",
       " ('masse passant', 8.461363306455567),\n",
       " ('passant Belgique', 8.461363306455567),\n",
       " ('Italie bloqué', 8.461363306455567),\n",
       " ('Italie bloquer', 8.461363306455567),\n",
       " ('bloquer lundi', 8.461363306455567),\n",
       " ('exonération impôt', 8.461363306455567),\n",
       " ('exonéré impôt', 8.461363306455567),\n",
       " ('blouse blanche', 8.461363306455567),\n",
       " ('blanche blouse', 8.461363306455567),\n",
       " ('Trump canons', 8.461363306455567),\n",
       " ('vote électronique', 8.461363306455567),\n",
       " ('choisiront tests', 8.461363306455567),\n",
       " ('tests choisiront', 8.461363306455567),\n",
       " ('complot chaînes', 8.461363306455567),\n",
       " ('tombé coma', 8.461363306455567),\n",
       " ('16h remis', 8.461363306455567),\n",
       " ('décédée 6h', 8.461363306455567),\n",
       " ('followers perd', 8.461363306455567),\n",
       " ('vide Bref', 8.461363306455567),\n",
       " ('prends page', 8.461363306455567),\n",
       " ('Issac joueur', 8.461363306455567),\n",
       " ('joueur NBA', 8.461363306455567),\n",
       " ('confirme enquête', 8.461363306455567),\n",
       " ('enquête circonstances', 8.461363306455567),\n",
       " ('circonstances cliquez', 8.461363306455567),\n",
       " ('Oui effectivement', 8.461363306455567),\n",
       " ('Sun accès', 8.461363306455567),\n",
       " ('accès source', 8.461363306455567),\n",
       " ('Allemagne journaux', 8.461363306455567),\n",
       " ('Calédonie décédée', 8.461363306455567),\n",
       " ('décédée domicile', 8.461363306455567),\n",
       " ('social EHPAD', 8.461363306455567),\n",
       " ('certificat médical', 8.461363306455567),\n",
       " ('médical atteste', 8.461363306455567),\n",
       " ('défunts considéré', 8.461363306455567),\n",
       " ('considéré potentiellement', 8.461363306455567),\n",
       " ('soins conservation', 8.461363306455567),\n",
       " ('débarrasser preuves', 8.461363306455567),\n",
       " ('preuves débarrasser', 8.461363306455567),\n",
       " ('fichier 90', 8.461363306455567),\n",
       " ('90 pourcents', 8.461363306455567),\n",
       " ('études statistiques', 8.461363306455567),\n",
       " ('trouvé parade', 8.461363306455567),\n",
       " ('parade recommande', 8.461363306455567),\n",
       " ('lundi Haute', 8.461363306455567),\n",
       " ('Denis agréé', 8.461363306455567),\n",
       " ('vital 1er', 8.461363306455567),\n",
       " ('Mélanie décédé', 8.461363306455567),\n",
       " ('décédé CHU', 8.461363306455567),\n",
       " ('arrêt cardiaque', 8.461363306455567),\n",
       " ('démission courageux', 8.461363306455567),\n",
       " ('classe noble', 8.461363306455567),\n",
       " ('promotion catimini', 8.461363306455567),\n",
       " ('entreprise Sanofi', 8.461363306455567),\n",
       " ('Sanofi chercheur', 8.461363306455567),\n",
       " ('intention balancer', 8.461363306455567),\n",
       " ('Nicolas Sarkozy', 8.461363306455567),\n",
       " ('échelle Europe', 8.461363306455567),\n",
       " ('députée européenne', 8.461363306455567),\n",
       " ('plan européen', 8.461363306455567),\n",
       " ('Nallet lance', 8.461363306455567),\n",
       " ('Michel Barnier', 8.461363306455567),\n",
       " ('Barnier expliquez', 8.461363306455567),\n",
       " ('bioMérieux expliquez', 8.461363306455567),\n",
       " ('autorisé cul', 8.461363306455567),\n",
       " ('inquiète gérer', 8.461363306455567),\n",
       " ('mettiez département', 8.461363306455567),\n",
       " ('Valérie département', 8.461363306455567),\n",
       " ('envoyez infos', 8.461363306455567),\n",
       " ('infos envoyez', 8.461363306455567),\n",
       " ('1941 neuf', 8.461363306455567),\n",
       " ('HP étudiante', 8.461363306455567),\n",
       " ('psychologue étonnes', 8.461363306455567),\n",
       " ('générale décervelé', 8.461363306455567),\n",
       " ('décervelé dirigent', 8.461363306455567),\n",
       " ('monte weekend', 8.461363306455567),\n",
       " ('quitte écrire', 8.461363306455567),\n",
       " ('dépassé Thaïlandaises', 8.461363306455567),\n",
       " ('Salim laibi', 8.461363306455567),\n",
       " ('condamné grosse', 8.461363306455567),\n",
       " ('grosse amende', 8.461363306455567),\n",
       " ('connaissais Salim', 8.461363306455567),\n",
       " ('voterai couilles', 8.461363306455567),\n",
       " ('cabinet médical', 8.461363306455567),\n",
       " ('rigole rire', 8.461363306455567),\n",
       " ('douleur critiquent', 8.461363306455567),\n",
       " ('ordonnance médicale', 8.461363306455567),\n",
       " ('produit viagra', 8.461363306455567),\n",
       " ('viagra violon', 8.461363306455567),\n",
       " ('maltraités bons', 8.461363306455567),\n",
       " ('promenade emmerdes', 8.461363306455567),\n",
       " ('Casablanca Corse', 8.461363306455567),\n",
       " ('liste célébrités', 8.461363306455567),\n",
       " ('célébrités balancé', 8.461363306455567),\n",
       " ('Bernard Tapie', 8.461363306455567),\n",
       " ('Tapie Alain', 8.461363306455567),\n",
       " ('Norman normal', 8.461363306455567),\n",
       " ('normal sécrétée', 8.461363306455567),\n",
       " ('traîneau chrome', 8.461363306455567),\n",
       " ('Belmondo citer', 8.461363306455567),\n",
       " ('buvait traite', 8.461363306455567),\n",
       " ('traite champagne', 8.461363306455567),\n",
       " ('Marbella venu', 8.461363306455567),\n",
       " ('vieillir Regarde', 8.461363306455567),\n",
       " ('Regarde Belmondo', 8.461363306455567),\n",
       " ('Belmondo devenue', 8.461363306455567),\n",
       " ('morfler bouquin', 8.461363306455567),\n",
       " ('séisme nomme', 8.461363306455567),\n",
       " ('nomme laisserai', 8.461363306455567),\n",
       " ('soin mousse', 8.461363306455567),\n",
       " ('livré espèces', 8.461363306455567),\n",
       " ('espèces maisons', 8.461363306455567),\n",
       " ('dilaté excuse', 8.461363306455567),\n",
       " ('Tom Cat', 8.461363306455567),\n",
       " ('Cat vend', 8.461363306455567),\n",
       " ('vend pièces', 8.461363306455567),\n",
       " ('pardonné violé', 8.461363306455567),\n",
       " ('jeunesse enfoirés', 8.461363306455567),\n",
       " ('enfoirés Frédéric', 8.461363306455567),\n",
       " ('Amos protège', 8.461363306455567),\n",
       " ('proxénétisme devait', 8.461363306455567),\n",
       " ('club Johnny', 8.461363306455567),\n",
       " ('Johnny Hallyday', 8.461363306455567),\n",
       " ('Hallyday Michel', 8.461363306455567),\n",
       " ('Sardou reconnu', 8.461363306455567),\n",
       " ('reconnu Johnny', 8.461363306455567),\n",
       " ('attaque diffamation', 8.461363306455567),\n",
       " ('voyou Franchement', 8.461363306455567),\n",
       " ('intéressait mouvement', 8.461363306455567),\n",
       " ('Marrakech Tanger', 8.461363306455567),\n",
       " ('Tanger pied-à-terre', 8.461363306455567),\n",
       " ('présidence Madame', 8.461363306455567),\n",
       " ('honneurs commis', 8.461363306455567),\n",
       " ('commis horreurs', 8.461363306455567),\n",
       " ('marchait nu', 8.461363306455567),\n",
       " ('portait numéro', 8.461363306455567),\n",
       " ('interview fier', 8.461363306455567),\n",
       " ('arrivé 1820', 8.461363306455567),\n",
       " ('sortent sort', 8.461363306455567),\n",
       " ('attaquent diffamation', 8.461363306455567),\n",
       " ('publiquement attaques', 8.461363306455567),\n",
       " ('attaques répercuter', 8.461363306455567),\n",
       " ('gênant aimerais', 8.461363306455567),\n",
       " ('cris femmes', 8.461363306455567),\n",
       " ('prends ligne', 8.461363306455567),\n",
       " ('fournir Hancock', 8.461363306455567),\n",
       " ('Hancock racheter', 8.461363306455567),\n",
       " ('champs faisais', 8.461363306455567),\n",
       " ('enculé volé', 8.461363306455567),\n",
       " ('envoyé livrer', 8.461363306455567),\n",
       " ('revenue langue', 8.461363306455567),\n",
       " ('cadeaux enfreindre', 8.461363306455567),\n",
       " ('rétroactive Normalement', 8.461363306455567),\n",
       " ('respecte lois', 8.461363306455567),\n",
       " ('Gainsbourg foutu', 8.461363306455567),\n",
       " ('Jean Gilles', 8.461363306455567),\n",
       " ('Guillou con', 8.461363306455567),\n",
       " ('mars avril', 8.461363306455567),\n",
       " ('avril prochain', 8.461363306455567),\n",
       " ('sida traite', 8.461363306455567),\n",
       " ('traite merdes', 8.461363306455567),\n",
       " ('The North', 8.461363306455567),\n",
       " ('couilles pensait', 8.461363306455567),\n",
       " ('routes sinueuses', 8.461363306455567),\n",
       " ('difficilement sort', 8.461363306455567),\n",
       " ('sort lol', 8.461363306455567),\n",
       " ('lol Normalement', 8.461363306455567),\n",
       " ('chance couru', 8.461363306455567),\n",
       " ('grosse tache', 8.461363306455567),\n",
       " ('chasse dorment', 8.461363306455567),\n",
       " ('Marseille Denis', 8.461363306455567),\n",
       " ('Denis bluetooth', 8.461363306455567),\n",
       " ('flics tombés', 8.461363306455567),\n",
       " ('descendu barres', 8.461363306455567),\n",
       " ('souviendrai sale', 8.461363306455567),\n",
       " ('venturi caïd', 8.461363306455567),\n",
       " ('venturi caille', 8.461363306455567),\n",
       " ('insultée traiter', 8.461363306455567),\n",
       " ('ego hyper', 8.461363306455567),\n",
       " ('hyper surdimensionné', 8.461363306455567),\n",
       " ('oses redirai', 8.461363306455567),\n",
       " ('copine pur', 8.461363306455567),\n",
       " ('premières analyses', 8.461363306455567),\n",
       " ('box Monsieur', 8.461363306455567),\n",
       " ('rarement saloperie', 8.461363306455567),\n",
       " ('vidé pourriture', 8.461363306455567),\n",
       " ('mimiques gestes', 8.461363306455567),\n",
       " ('arrêt double', 8.461363306455567),\n",
       " ('supprimer fréquence', 8.461363306455567),\n",
       " ('braquage informer', 8.461363306455567),\n",
       " ('informer cours', 8.461363306455567),\n",
       " ('passait télésurveillance', 8.461363306455567),\n",
       " ('happy fenêtre', 8.461363306455567),\n",
       " ('fenêtre chambre', 8.461363306455567),\n",
       " ('consommable tué', 8.461363306455567),\n",
       " ('prenne cocaïne', 8.461363306455567),\n",
       " ('cocaïne inévitable', 8.461363306455567),\n",
       " ('enfer terminer', 8.461363306455567),\n",
       " ('terminer obligatoire', 8.461363306455567),\n",
       " ('dégueulasses nu', 8.461363306455567),\n",
       " ('tourné présente', 8.461363306455567),\n",
       " ('messages voluer', 8.461363306455567),\n",
       " ('Dreno chrome', 8.461363306455567),\n",
       " ('gâteau adresse', 8.461363306455567),\n",
       " ('kidnappé achète', 8.461363306455567),\n",
       " ('crève organes', 8.461363306455567),\n",
       " ('oublier vendre', 8.461363306455567),\n",
       " ('vendre beaux', 8.461363306455567),\n",
       " ('table bourrés', 8.461363306455567),\n",
       " ('cracher fallu', 8.461363306455567),\n",
       " ('révélation YouTube', 8.461363306455567),\n",
       " ('ski apparence', 8.461363306455567),\n",
       " ('apparence paraît', 8.461363306455567),\n",
       " ('garder Pass', 8.461363306455567),\n",
       " ('recherche rapide', 8.461363306455567),\n",
       " ('rapide molécule', 8.461363306455567),\n",
       " ('55 25', 8.461363306455567),\n",
       " ('25 Canada', 8.461363306455567),\n",
       " ('soins intensifs', 8.461363306455567),\n",
       " ('136 Alizée', 8.461363306455567),\n",
       " ('mucormycose patients', 8.461363306455567),\n",
       " ('patients guéri', 8.461363306455567),\n",
       " ('développement patients', 8.461363306455567),\n",
       " ('considéré substance', 8.461363306455567),\n",
       " ('profil constant', 8.461363306455567),\n",
       " ('défenses immunitaires', 8.461363306455567),\n",
       " ('Chante profil', 8.461363306455567),\n",
       " ('dessins animés', 8.461363306455567),\n",
       " ('messages subliminaux', 8.461363306455567),\n",
       " ('propane pourri', 8.461363306455567),\n",
       " ('cadre Clarence', 8.461363306455567),\n",
       " ('éteins mode', 8.461363306455567),\n",
       " ('monte instants', 8.461363306455567),\n",
       " ('instants garder', 8.461363306455567),\n",
       " ('organisations tousse', 8.461363306455567),\n",
       " ('karting ville', 8.461363306455567),\n",
       " ('rats anciens', 8.461363306455567),\n",
       " ('anciens fragiles', 8.461363306455567),\n",
       " ('pure manipulation', 8.461363306455567),\n",
       " ('manipulation triste', 8.461363306455567),\n",
       " ('pâtes Annick', 8.461363306455567),\n",
       " ('engloutis grandi', 8.461363306455567),\n",
       " ('Enfant soleil', 8.461363306455567),\n",
       " ('soleil parcours', 8.461363306455567),\n",
       " ('tombe religions', 8.461363306455567),\n",
       " ('pleut souris', 8.461363306455567),\n",
       " ('discours diabolique', 8.461363306455567),\n",
       " ('infirmier médecins', 8.461363306455567),\n",
       " ('médecins mobilisés', 8.461363306455567),\n",
       " ('présenter lentes', 8.461363306455567),\n",
       " ('crise 2009', 8.461363306455567),\n",
       " ('forcé 450', 8.461363306455567),\n",
       " ('h1n touché', 8.461363306455567),\n",
       " ('panda facebook', 8.461363306455567),\n",
       " ('grippale signal', 8.461363306455567),\n",
       " ('signal adresser', 8.461363306455567),\n",
       " ('hein tweet', 8.461363306455567),\n",
       " ('30.000 marre', 8.461363306455567),\n",
       " ('Bachelot connaissez', 8.461363306455567),\n",
       " ('connaissez production', 8.461363306455567),\n",
       " ('adapté nécessités', 8.461363306455567),\n",
       " ('citant sondages', 8.461363306455567),\n",
       " ('sondages inquiétant', 8.461363306455567),\n",
       " ('Jean survive', 8.461363306455567),\n",
       " ('covid artificiellement', 8.461363306455567),\n",
       " ('maîtrisez chemin', 8.461363306455567),\n",
       " ('creuser isation', 8.461363306455567),\n",
       " ('Cazarre creuser', 8.461363306455567),\n",
       " ('creuser nombres', 8.461363306455567),\n",
       " ('Nuremberg gala', 8.461363306455567),\n",
       " ('gala respecte', 8.461363306455567),\n",
       " ('conscience appuyez', 8.461363306455567),\n",
       " ('appuyez simplement', 8.461363306455567),\n",
       " ('simplement drapeau', 8.461363306455567),\n",
       " ('Sylla poches', 8.461363306455567),\n",
       " ('vouloir apprenez', 8.461363306455567),\n",
       " ('apprenez mentir', 8.461363306455567),\n",
       " ('mentir électrons', 8.461363306455567),\n",
       " ('prenez ouvrez', 8.461363306455567),\n",
       " ('masques expériences', 8.461363306455567),\n",
       " ('expériences quasar', 8.461363306455567),\n",
       " ('synagogue Satan', 8.461363306455567),\n",
       " ('rotavirus expériences', 8.461363306455567),\n",
       " ('expériences fœtus', 8.461363306455567),\n",
       " ('fœtus avorter', 8.461363306455567),\n",
       " ('pro tueur', 8.461363306455567),\n",
       " ('tueur brillant', 8.461363306455567),\n",
       " ('pizza dure', 8.461363306455567),\n",
       " ('dure dissimulé', 8.461363306455567),\n",
       " ('fond vertu', 8.461363306455567),\n",
       " ('connerie demander', 8.461363306455567),\n",
       " ('demander absolution', 8.461363306455567),\n",
       " ('Charny encourage', 8.461363306455567),\n",
       " ('encourage aidera', 8.461363306455567),\n",
       " ('aidera dénonce', 8.461363306455567),\n",
       " ('dénonce frappe', 8.461363306455567),\n",
       " ('frappe état', 8.461363306455567),\n",
       " ('tendre Sylla', 8.461363306455567),\n",
       " ('reviennent enfer', 8.461363306455567),\n",
       " ('collège fond', 8.461363306455567),\n",
       " ('avancer sincère', 8.461363306455567),\n",
       " ('dimension foi', 8.461363306455567),\n",
       " ('foi chrétienne', 8.461363306455567),\n",
       " ('démasquer golf', 8.461363306455567),\n",
       " ('golf salon', 8.461363306455567),\n",
       " ('the ghost', 8.461363306455567),\n",
       " ('town République', 8.461363306455567),\n",
       " ('République ripoux', 8.461363306455567),\n",
       " ('Monsieur Ploncard', 8.461363306455567),\n",
       " ('aspirais Seb', 8.461363306455567),\n",
       " ('Seb Confluence', 8.461363306455567),\n",
       " ('assis cramponnez', 8.461363306455567),\n",
       " ('gestes médiatiques', 8.461363306455567),\n",
       " ('amusantes constaté', 8.461363306455567),\n",
       " ('constaté lisant', 8.461363306455567),\n",
       " ('Tanya cinéma', 8.461363306455567),\n",
       " ('décennie progrès', 8.461363306455567),\n",
       " ('progrès accompli', 8.461363306455567),\n",
       " ('pourrons réduire', 8.461363306455567),\n",
       " ('réduire 9', 8.461363306455567),\n",
       " ('meurent année', 8.461363306455567),\n",
       " ('année moitié', 8.461363306455567),\n",
       " ('révolution industrielle', 8.461363306455567),\n",
       " ('industrielle forcer', 8.461363306455567),\n",
       " ('modifie génome', 8.461363306455567),\n",
       " ('génome modifié', 8.461363306455567),\n",
       " ('forcé évolutions', 8.461363306455567),\n",
       " ('évolutions comportement', 8.461363306455567),\n",
       " ('programmer propre', 8.461363306455567),\n",
       " ...]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pointwise Mutual Information (PMI)\n",
    "The TDNA paper uses PMI to determine which ngrams to include. \n",
    "\n",
    "$$\n",
    "PMI(a,b) = log\\frac{p(a,b)}{p(a)p(b)} = log\\frac{p(a|b)}{p(a)} = log\\frac{p(b|a)}{p(b)}\n",
    "$$\n",
    "\n",
    "For each sentence $X$ with tokens  $x_1, x_2 .... x_t$   \n",
    "Find ngrams with a $PMI$ score above some threshold   \n",
    "Generate lexicon $L$ where each ngram appears with a frequency of at least $f$   \n",
    "For each token in $X$, check if a substring exists in $L$, If so, extract the ngram from $L$ to form $S$ with ngrams $s_1, s_2, ... s_k$   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### To use PMI, set pmi=True. Otherwise, the script will use simple frequencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "importlib.reload(vu)\n",
    "\n",
    "t = tqdm(langs)\n",
    "for lang in t:\n",
    "    t.set_description(\"processing language: \"+lang[1]+, refresh=True)\n",
    "    data = lang[0]\n",
    "    language = lang[1]\n",
    "    model=lang[2]\n",
    "    stopwords = lang[3]\n",
    "    max_n = 32768\n",
    "        \n",
    "    ngrams_list = vu.getNgramsSpacy(data['text'],\n",
    "                                     stopwords=stopwords,\n",
    "                                     spacy_model=model,\n",
    "                                     LLMvocab=xlm_roberta_vocab,\n",
    "                                     language=language,\n",
    "                                     max_n=max_n,\n",
    "                                     pmi=True)\n",
    "\n",
    "    with open('../data/ngrams/'+language+'_ngrams_'+str(max_n)+'.tsv', 'w') as f:\n",
    "        for item in ngrams_list:\n",
    "            f.write(\"%s\\t%s\\n\" % (item[0], item[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make embeddings\n",
    "Before proceeding with the next steps, make sure to train fasttext models and generate the embeddings for the ngrams"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TAPT-n/fasttext-train-multilingual.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['fr', 'de', 'es', 'hi', 'pt', 'ru', 'sv', 'tr']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "languages = ['fr','de','es','hi','pt','ru','sv','tr','zh','ar']\n",
    "languages[:-2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fr 2318\n",
      "de 32768\n",
      "es 6114\n",
      "hi 2392\n",
      "pt 11930\n",
      "ru 3097\n",
      "sv 3977\n",
      "tr 8830\n"
     ]
    }
   ],
   "source": [
    "# reduce the number of ngrams by using only ones with frequency > 100\n",
    "# combine all languages for training (skip chinese and arabic for now)\n",
    "\n",
    "languages = ['fr','de','es','hi','pt','ru','sv','tr','zh','ar']\n",
    "\n",
    "all_ngrams = pd.DataFrame(columns=['ngram','count'])\n",
    "all_embeddings = np.empty([0, 768])\n",
    "\n",
    "for lang in languages[:-2]:\n",
    "    df = pd.read_csv('../data/ngrams/'+lang+'_ngrams_32768.tsv',sep=\"\\t\",header=None,names=['ngram','count'])\n",
    "    embeddings = np.load('../data/ngrams/'+lang+'_ngrams_32768.npy')\n",
    "    \n",
    "    size = len(df[df['count']>100])\n",
    "    print(lang, size)\n",
    "    short_ngrams = df[:size]\n",
    "    short_embeddings = embeddings[:size]\n",
    "    \n",
    "    all_embeddings = np.concatenate((all_embeddings, short_embeddings))\n",
    "    all_ngrams = pd.concat([all_ngrams,short_ngrams])\n",
    "    \n",
    "    # save to file\n",
    "    short_ngrams.to_csv('../data/ngrams/short/'+lang+'_ngrams_'+str(size)+'.tsv',sep=\"\\t\",header=None,index=None)\n",
    "    np.save('../data/ngrams/short/'+lang+'_ngrams_'+str(size)+'.npy',short_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add the english embeddings\n",
    "df = pd.read_csv('../data/english_snippet_graph_matches_100k_ngrams_32768.tsv',sep=\"\\t\",header=None,names=['ngram','count'])\n",
    "embeddings = np.load('../models/english_snippet_graph_matches_100k_fasttext_3.7.22_768.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "size = len(df[df['count']>5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8772"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "71426"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "short_ngrams = df[:size]\n",
    "short_embeddings = embeddings[:size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_embeddings = np.concatenate((all_embeddings, short_embeddings))\n",
    "all_ngrams = pd.concat([all_ngrams,short_ngrams])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "80198\n",
      "80198\n"
     ]
    }
   ],
   "source": [
    "print(len(all_embeddings))\n",
    "print(len(all_ngrams))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_ngrams.to_csv('../data/ngrams/short/xlm_ngrams_768.tsv',sep=\"\\t\",header=None, index=None)\n",
    "np.save('../data/ngrams/short/xlm_ngrams_768.npy',all_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# concatenate and shuffle all the data (except chinese and arabic)\n",
    "!cat ../data/all-transcripts/transcripts-all-de.csv \\\n",
    "../data/all-transcripts/transcripts-all-es.csv \\\n",
    "../data/all-transcripts/transcripts-all-fr.csv \\\n",
    "../data/all-transcripts/transcripts-all-hi.csv \\\n",
    "../data/all-transcripts/transcripts-all-pt.csv \\\n",
    "../data/all-transcripts/transcripts-all-ru.csv \\\n",
    "../data/all-transcripts/transcripts-all-sv.csv \\\n",
    "../data/all-transcripts/transcripts-all-tr.csv \\\n",
    "../data/english_snippet_graph_matches_100k.csv \\\n",
    "| shuf > ../data/all-transcripts/transcripts-all.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# take a sample of of the transcripts data\n",
    "!head -n 200 ../data/all-transcripts/transcripts-all.csv > ../data/all-transcripts/sample.csv "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If the wc of ngrams doesn't equal the length of the embeddings, it means some ngrams have the new line character. Then we can't use simple pandas csv write.\n",
    "# Instead we remove the '\\n' charcter and write to file the python way\n",
    "### all_ngrams.to_csv('../data/ngrams/short/xlm_ngrams_768.tsv',sep=\"\\t\",header=None,index=None)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "with open('../data/ngrams/short/xlm_ngrams_768.tsv', \"w\", encoding=\"utf-8\") as fout:\n",
    "    \n",
    "    for row in all_ngrams.iterrows():\n",
    "        ngram, freq = row[1]['ngram'], row[1]['count']\n",
    "        ngram = ngram.replace('\\n','')\n",
    "        fout.write(\"{}\\t{}\\n\".format(ngram, freq))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "80198 ../data/ngrams/short/xlm_ngrams_768.tsv\n"
     ]
    }
   ],
   "source": [
    "!wc -l ../data/ngrams/short/xlm_ngrams_768.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# preprocessing the data\n",
    "* Remove the quotation marks at the start and end of each sentence\n",
    "* Split into chunks so we can use linebyline transformers function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tests on sample data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# first remove the first 3 quotations\n",
    "!cut -c 4- ../data/transcripts/sample.csv > ../data/transcripts/sample2.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# then remove the last quotation by spliting on quotation mark - kind of a work-around\n",
    "!cut -d '\"' -f 1 ../data/transcripts/sample2.csv > ../data/transcripts/sample.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spielt die Wahrheit eine Rolle nicht zu Gruppen wie Black lies Mörder das ist aus vielen Gründen tragisch nicht zuletzt weil dadurch schwarz sie ihr Leben verlieren wenn es um das Thema amerikanische Polizei schwarze und die tödliche Anwendung von Gewalt geht wissen wir folgendes eine kürzlich von Louis James Forscherin an der Washington State University durchgeführte Studie über tödliche Gewalt ergab dass Polizeibeamte in simulierten Bedrohungsszenarien mit geringerer Wahrscheinlichkeit auf unbewaffnete schwarze Verdächtige schießen als auf unbewaffnete weiße oder Hispano Amerikaner der Harvard Wirtschaftsprofessor Ronald freier analysierte mehr als 1000 Erschießungen durch Polizeibeamte im ganzen Land er kam zu dem Schluss dass es bei Erschießungen durch die Polizei keinerlei Anzeichen für eine rassistische Voreingenommenheit gibt in Houston stellte er fest das schwarze mit 24% geringerer Wahrscheinlichkeit von Polizisten erschossen wurden als weiße obwohl die Verdächtigen bewaffnet oder gewalttätig waren spielt die Wahrheit eine Rolle eine Analyse der Schießerei Datenbank der Washington Post und der bundeskriminal Statistik zeigt das ganze 12% aller weißen und Hispanics die an Tötungsdelikten sterben von Polizisten getötet werden im Gegensatz dazu werden nur 4% der schwarzen Mordopfer von Polizisten getötet aber es ist nicht ein Zeichen von Voreingenommenheit das schwarze 26% der Opfer von Tötung durch Polizisten ausmachen aber nur 13% der nationalen Bevölkerung das ist es nicht und der gesunde Menschenverstand lignal Warum Erschießungen durch die Polizei treten häufiger dort auf wo Beamte mit bewaffneten oder sich gewaltsam widersetzen den Verdächtigen konfrontiert werden diese Verdächtigen sind überproportional häufig schwarz lauter jüngsten Studie des Justizministeriums wurden schwarze obwohl sie nur etwa 15% der Bevölkerung in den 75 größten Bezirken der USA ausmacht für 62% alle Raubüberfälle 57 % aller Morde und 45 % aller Körperverletzung angeklagt in New York City beginnt schwarze über dreiviertel aller Schießereien obwohl sie nur 23% der Bevölkerung der Stadt ausmachen weiße hingegen begehen weniger als 2% aller Schießereien in der Stadt obwohl sie 34% der Bevölkerung ausmachen Kriminalität Unterschiede in New York wiederholen sich in praktisch jeder Stadt Amerikas in der Rassen Vielfalt herrscht das eigentliche Problem mit dem die schwarzen Gemeinde den Innenstädten heute konfrontiert sind sind nicht die Polizei sondern die Kriminellen im Jahr 2014 wurden über 6.000 schwarze ermordet mehr als alle weißen und hispanischen Mordopfer zusammen wer tötet sie nicht die Polizei und keine weißen Zivilisten sondern andere schwarze tatsächlich ist die Wahrscheinlichkeit dass ein Polizeibeamter von einem schwarzen getötet wird 18 einhalb mal höher als die Wahrscheinlichkeit dass ein unbewaffneter schwarzer von einem Polizeibeamten getötet wird wenn die Polizei morgen jeglichen Einsatz von tödlicher Gewalt beenden würde hätte dies einen vernachlässigbaren Einfluss auf die Mordrate von schwarzen Chicago wurden allein in den ersten sechs einhalb Monaten des Jahres 2016 über 2300 Menschen erschossen das entspricht an manchen Wochenenden einer Schießerei pro Stunde die überwiegende Mehrheit der Opfer waren schwarze im gleichen Zeitraum erschoss die Chicagoer Polizei 12 Menschen alle bewaffnet und gefährlich das ist die Hälfte von einem Prozent aller Schießereien spielt die Wahrheit eine Rolle falls ja hier ist eine Wahrheit die es wert ist über sie nachzudenken es gibt keine Behörde die sich mehr der Behauptung verschrieben hat das schwarze Leben eine Rolle spielen als die Polizei die proaktive polizeiliche Revolution die Mitte der 1990 er Jahre begann hat die innerstädtische Mordrate drastisch gesenkt und Zehntausende von schwarzen Leben gerettet leider ist dieser Rückgang der Kriminalität jetzt in Gefahr wie ich in meinem Buch der Krieg gegen die Polizei schreibe ziehen sich die Polizeibeamten von der proaktiven Polizeiarbeit in den schwarzen Vierteln zurück dank der falschen Erzählung dass Polizeibeamte mit mörderischer Voreingenommenheit infiziert sein infolgedessen nimmt die Gewaltkriminalität zu in Städten mit einer großen schwarzen Bevölkerung sind die Morde im Jahr 2015 zwischen 54% in Washington DC und 90 % in Cleveland angestiegen insgesamt stiegen die Morde in den 56 größten Städten des Landes im Jahr 2015 um 17% ein fast beispiellose Anstieg in einem Jahr viele Gesetzestreue Bewohner von Gebieten mit hoher Kriminalität betteln bei der Polizei um die Aufrechterhaltung der Ordnung genau die Art von Polizeiarbeit die von der acr.eu progressiven Politikern und dem Obama Justizministerium als rassistisch angeprangert wird das ist tragisch denn wenn die Polizei von einer proaktiven Polizeiarbeit Abstand nimmt kommen schwarze ums Leben verloren wegen eines Mythos die beste Forschung und die besten Daten kommen zu diesem Schluss es gibt keine Beweise dafür dass die Polizei schwarze tötet nur weil sie schwarz sind ich bin härter McDonald für Prager university \n",
      "Deutschlandfunk 21 Uhr die Nachrichten die Staaten des Ölkartell zopick haben sich nach zehn Verhandlungen darauf geeinigt die Förderung des Rohstoffs für zwei Monate zu reduzieren das geht aus übereinstimmenden mitteilungen mehrerer Mitgliedstaaten hervor in den Monaten Mai und Juni soll 9,7 Millionen Barrel Öl weniger pro Tag gefördert werden die Menge liegt geringfügig unter der die zunächst in den Verhandlungen der OPEC Staaten zur Debatte stand Ziel der Länder ist es den fallenden Ölpreisen auf dem Weltmarkt entgegenzuwirken und das Angebot zu verknappen in Italien und Frankreich sind die Zahl der täglichen Todesfälle infolge von Corona Infektionen aus Italien wurden zum ersten Mal seit 3 Wochen weniger als 500 Todesopfer täglich registriert der Zivilschutz in Rom teilte am Abend mit in den vergangenen 24 Stunden sein 431 Menschen nach einer Infektion mit dem Virus gestorben insgesamt verloren während der Pandemie knapp 20.000 Menschen in Italien ihr Leben infolge einer covid-19 Erkrankungen die französischen Behörden meldeten 315 tote innerhalb eines Tages auch dies ist ein leichter Rückgang im Vergleich zum Vortag seit Anfang März starben infolge von covid Infektionen knapp 14.000 400 Menschen in Frankreich etwa ein Drittel davon waren Bewohner von alten und Pflegeheimen bei den Corona Angaben gibt es immer wieder Abweichungen einer der Gründe ist das nicht alle verfügbaren Datenbanken permanent und zur gleichen Zeit aktualisiert werden außer in den teilweise unterschiedliche Quellen verwendet  in der Debatte über die Einschränkungen des öffentlichen Lebens wird über die Bedingungen für eine schrittweise Lockerung diskutiert mit Blick auf die deutsche Wirtschaft hält Bundesgesundheitsminister Spahn eine allmähliche Rückkehr zur Normalität für möglich sofern bestimmte Bronson zeigten dass sie die erforderlichen Hygiene und Abstands Regeln durchsetzen könnten knifflig bleibe es für Schulen und Kindergärten sagt die CDU Politiker bild.tv nach den Worten von Nordrhein-Westfalens Ministerpräsident Laschet stehen viele kleine vorsichtige Schritte bevor in einer Fernsehansprache forderte der CDU-Politiker einen Fahrplan der den Weg in eine verantwortungsvolle Normalität zeige der deutsche Städte und Gemeindebund sieht in flächendeckenden Corona Test eine Voraussetzung für Lockerungen der Schutzmaßnahmen Hauptgeschäftsführer Landsberg PDT in den Zeitungen der funke-mediengruppe dafür ein bundesweit einheitliches Test und Meldesystem aufzubauen außerdem müssten die T  Kapazitäten deutlich ausgebaut werden Papst franziskus hat in seiner Osterbotschaft zu weltweitem Zusammenhalt aufgerufen die Menschheit werde zurzeit auf eine harte Probe gestellt erklärte das Oberhaupt der katholischen Kirche in der traditionellen Ostermesse diese fand wegen der Ausbreitung des Coronavirus in einem fast menschenleeren Petersdom Stadt franziskus betonte die Krise erlaube weder Gleichgültigkeit noch Egoismus erforderte daher einen Schuldenerlass für ärmere Staaten diese seien kaum gerüstet um sich gegen die Pandemie zu stemmen der Papst spendete außerdem den traditionellen Segen Urbi et Orbi der Stadt und dem Erdkreis normalerweise findet die Ostermesse vor zehntausenden Besuchern auf dem Petersplatz Stadt zur Eindämmung des Coronavirus werden in diesem Jahr jedoch alle Osterfeierlichkeiten ohne die Anwesenheit von Gläubigen und Pilgern abgehalten die besten werden vom wa  wie kann im Internet und von vielen Fernsehsendern weltweit übertragen in Israel bekommt Oppositionsführer ganz nicht mehr Zeit um doch noch eine Regierung zu bilden Stadtpräsident rieflin teilte mit eine Fristverlängerung um weitere zwei Wochen sei unter den gegebenen Umständen nicht möglich damit dürften in der Nacht zu Dienstag die vierwöchigen Fristen zur Regierungsbildung ablaufen ganz der das Mitte Bündnis Blau-Weiß anführt hatte die Verlängerung beantragt nachdem Verhandlungen mit den rechtskonservativen Nico des amtierenden Ministerpräsidenten Netanjahu erfolglos verlaufen waren das Wetter in der Nacht im Südwesten und an den Alpen anfangs noch Schauer sonst in der Südhälfte meist gering bewölkt oder klar im Norden bewölkt mit schaue artige im Regen Tiefstwerte 9 bis 3 Grad am Ostermontag im Norden weitgehend trocken von der Mitte allmählich nach Süden aus  Bewölkung mit schaumartige im Regen vereinzelt auch gewittrig in Hochlagen der Mittelgebirge ist Schnee möglich Werte von 8 Grad an der Ostsee bis 22 Grad am Oberrhein das waren die Nachrichten \n"
     ]
    }
   ],
   "source": [
    "# let's see if it worked:\n",
    "!head -n 2 ../data/transcripts/sample.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split into chunks:\n",
    "# insert a line break every n charcters\n",
    "# https://man.openbsd.org/fold\n",
    "!fold -bs -w 3000 ../data/transcripts/sample.csv > sample_chunked.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spielt die Wahrheit eine Rolle nicht zu Gruppen wie Black lies Mörder das ist aus vielen Gründen tragisch nicht zuletzt weil dadurch schwarz sie ihr Leben verlieren wenn es um das Thema amerikanische Polizei schwarze und die tödliche Anwendung von Gewalt geht wissen wir folgendes eine kürzlich von Louis James Forscherin an der Washington State University durchgeführte Studie über tödliche Gewalt ergab dass Polizeibeamte in simulierten Bedrohungsszenarien mit geringerer Wahrscheinlichkeit auf unbewaffnete schwarze Verdächtige schießen als auf unbewaffnete weiße oder Hispano Amerikaner der Harvard Wirtschaftsprofessor Ronald freier analysierte mehr als 1000 Erschießungen durch Polizeibeamte im ganzen Land er kam zu dem Schluss dass es bei Erschießungen durch die Polizei keinerlei Anzeichen für eine rassistische Voreingenommenheit gibt in Houston stellte er fest das schwarze mit 24% geringerer Wahrscheinlichkeit von Polizisten erschossen wurden als weiße obwohl die Verdächtigen bewaffnet oder gewalttätig waren spielt die Wahrheit eine Rolle eine Analyse der Schießerei Datenbank der Washington Post und der bundeskriminal Statistik zeigt das ganze 12% aller weißen und Hispanics die an Tötungsdelikten sterben von Polizisten getötet werden im Gegensatz dazu werden nur 4% der schwarzen Mordopfer von Polizisten getötet aber es ist nicht ein Zeichen von Voreingenommenheit das schwarze 26% der Opfer von Tötung durch Polizisten ausmachen aber nur 13% der nationalen Bevölkerung das ist es nicht und der gesunde Menschenverstand lignal Warum Erschießungen durch die Polizei treten häufiger dort auf wo Beamte mit bewaffneten oder sich gewaltsam widersetzen den Verdächtigen konfrontiert werden diese Verdächtigen sind überproportional häufig schwarz lauter jüngsten Studie des Justizministeriums wurden schwarze obwohl sie nur etwa 15% der Bevölkerung in den 75 größten Bezirken der USA ausmacht für 62% alle Raubüberfälle 57 % aller Morde und 45 % aller Körperverletzung angeklagt in New York City beginnt schwarze über dreiviertel aller Schießereien obwohl sie nur 23% der Bevölkerung der Stadt ausmachen weiße hingegen begehen weniger als 2% aller Schießereien in der Stadt obwohl sie 34% der Bevölkerung ausmachen Kriminalität Unterschiede in New York wiederholen sich in praktisch jeder Stadt Amerikas in der Rassen Vielfalt herrscht das eigentliche Problem mit dem die schwarzen Gemeinde den Innenstädten heute konfrontiert sind sind nicht die Polizei sondern die Kriminellen im Jahr 2014 wurden über 6.000 schwarze ermordet mehr als alle weißen und hispanischen Mordopfer zusammen wer tötet sie nicht die Polizei und keine weißen Zivilisten sondern andere schwarze tatsächlich ist die Wahrscheinlichkeit dass ein Polizeibeamter von einem schwarzen getötet wird 18 einhalb mal höher als die Wahrscheinlichkeit dass ein unbewaffneter schwarzer von einem Polizeibeamten getötet wird wenn die Polizei morgen jeglichen Einsatz von \n",
      "tödlicher Gewalt beenden würde hätte dies einen vernachlässigbaren Einfluss auf die Mordrate von schwarzen Chicago wurden allein in den ersten sechs einhalb Monaten des Jahres 2016 über 2300 Menschen erschossen das entspricht an manchen Wochenenden einer Schießerei pro Stunde die überwiegende Mehrheit der Opfer waren schwarze im gleichen Zeitraum erschoss die Chicagoer Polizei 12 Menschen alle bewaffnet und gefährlich das ist die Hälfte von einem Prozent aller Schießereien spielt die Wahrheit eine Rolle falls ja hier ist eine Wahrheit die es wert ist über sie nachzudenken es gibt keine Behörde die sich mehr der Behauptung verschrieben hat das schwarze Leben eine Rolle spielen als die Polizei die proaktive polizeiliche Revolution die Mitte der 1990 er Jahre begann hat die innerstädtische Mordrate drastisch gesenkt und Zehntausende von schwarzen Leben gerettet leider ist dieser Rückgang der Kriminalität jetzt in Gefahr wie ich in meinem Buch der Krieg gegen die Polizei schreibe ziehen sich die Polizeibeamten von der proaktiven Polizeiarbeit in den schwarzen Vierteln zurück dank der falschen Erzählung dass Polizeibeamte mit mörderischer Voreingenommenheit infiziert sein infolgedessen nimmt die Gewaltkriminalität zu in Städten mit einer großen schwarzen Bevölkerung sind die Morde im Jahr 2015 zwischen 54% in Washington DC und 90 % in Cleveland angestiegen insgesamt stiegen die Morde in den 56 größten Städten des Landes im Jahr 2015 um 17% ein fast beispiellose Anstieg in einem Jahr viele Gesetzestreue Bewohner von Gebieten mit hoher Kriminalität betteln bei der Polizei um die Aufrechterhaltung der Ordnung genau die Art von Polizeiarbeit die von der acr.eu progressiven Politikern und dem Obama Justizministerium als rassistisch angeprangert wird das ist tragisch denn wenn die Polizei von einer proaktiven Polizeiarbeit Abstand nimmt kommen schwarze ums Leben verloren wegen eines Mythos die beste Forschung und die besten Daten kommen zu diesem Schluss es gibt keine Beweise dafür dass die Polizei schwarze tötet nur weil sie schwarz sind ich bin härter McDonald für Prager university \n"
     ]
    }
   ],
   "source": [
    "!head -n 2 sample_chunked.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## preprocess all the transcripts\n",
    "Transformers Datasets actually has python code that will chunk the data, but... it reads the entire dataset into memory and then uses slices to chunk it. The advantage of using transformers is that it will chunk on tokens. But.. that's a pretty memory intensive task, and I'm betting on linux even though the chunks will someties need to be padded, and sometimes will be truncated.\n",
    "\n",
    "The dataset to use for mlm will be: `../data/transcripts/transcripts-all-chunked.csv`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cut -c 4- ../data/transcripts/transcripts-all.csv > ../data/transcripts/transcripts-all-4.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cut -d '\"' -f 1 ../data/transcripts/transcripts-all-4.csv > ../data/transcripts/transcripts-all.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "!fold -bs -w 800 ../data/transcripts/transcripts-all.csv > ../data/transcripts/transcripts-all-chunked.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spielt die Wahrheit eine Rolle nicht zu Gruppen wie Black lies Mörder das ist aus vielen Gründen tragisch nicht zuletzt weil dadurch schwarz sie ihr Leben verlieren wenn es um das Thema amerikanische Polizei schwarze und die tödliche Anwendung von Gewalt geht wissen wir folgendes eine kürzlich von Louis James Forscherin an der Washington State University durchgeführte Studie über tödliche Gewalt ergab dass Polizeibeamte in simulierten Bedrohungsszenarien mit geringerer Wahrscheinlichkeit auf unbewaffnete schwarze Verdächtige schießen als auf unbewaffnete weiße oder Hispano Amerikaner der Harvard Wirtschaftsprofessor Ronald freier analysierte mehr als 1000 Erschießungen durch Polizeibeamte im ganzen Land er kam zu dem Schluss dass es bei Erschießungen durch die Polizei \n",
      "keinerlei Anzeichen für eine rassistische Voreingenommenheit gibt in Houston stellte er fest das schwarze mit 24% geringerer Wahrscheinlichkeit von Polizisten erschossen wurden als weiße obwohl die Verdächtigen bewaffnet oder gewalttätig waren spielt die Wahrheit eine Rolle eine Analyse der Schießerei Datenbank der Washington Post und der bundeskriminal Statistik zeigt das ganze 12% aller weißen und Hispanics die an Tötungsdelikten sterben von Polizisten getötet werden im Gegensatz dazu werden nur 4% der schwarzen Mordopfer von Polizisten getötet aber es ist nicht ein Zeichen von Voreingenommenheit das schwarze 26% der Opfer von Tötung durch Polizisten ausmachen aber nur 13% der nationalen Bevölkerung das ist es nicht und der gesunde Menschenverstand lignal Warum Erschießungen \n",
      "durch die Polizei treten häufiger dort auf wo Beamte mit bewaffneten oder sich gewaltsam widersetzen den Verdächtigen konfrontiert werden diese Verdächtigen sind überproportional häufig schwarz lauter jüngsten Studie des Justizministeriums wurden schwarze obwohl sie nur etwa 15% der Bevölkerung in den 75 größten Bezirken der USA ausmacht für 62% alle Raubüberfälle 57 % aller Morde und 45 % aller Körperverletzung angeklagt in New York City beginnt schwarze über dreiviertel aller Schießereien obwohl sie nur 23% der Bevölkerung der Stadt ausmachen weiße hingegen begehen weniger als 2% aller Schießereien in der Stadt obwohl sie 34% der Bevölkerung ausmachen Kriminalität Unterschiede in New York wiederholen sich in praktisch jeder Stadt Amerikas in der Rassen Vielfalt \n",
      "herrscht das eigentliche Problem mit dem die schwarzen Gemeinde den Innenstädten heute konfrontiert sind sind nicht die Polizei sondern die Kriminellen im Jahr 2014 wurden über 6.000 schwarze ermordet mehr als alle weißen und hispanischen Mordopfer zusammen wer tötet sie nicht die Polizei und keine weißen Zivilisten sondern andere schwarze tatsächlich ist die Wahrscheinlichkeit dass ein Polizeibeamter von einem schwarzen getötet wird 18 einhalb mal höher als die Wahrscheinlichkeit dass ein unbewaffneter schwarzer von einem Polizeibeamten getötet wird wenn die Polizei morgen jeglichen Einsatz von tödlicher Gewalt beenden würde hätte dies einen vernachlässigbaren Einfluss auf die Mordrate von schwarzen Chicago wurden allein in den ersten sechs einhalb Monaten des Jahres 2016 \n",
      "über 2300 Menschen erschossen das entspricht an manchen Wochenenden einer Schießerei pro Stunde die überwiegende Mehrheit der Opfer waren schwarze im gleichen Zeitraum erschoss die Chicagoer Polizei 12 Menschen alle bewaffnet und gefährlich das ist die Hälfte von einem Prozent aller Schießereien spielt die Wahrheit eine Rolle falls ja hier ist eine Wahrheit die es wert ist über sie nachzudenken es gibt keine Behörde die sich mehr der Behauptung verschrieben hat das schwarze Leben eine Rolle spielen als die Polizei die proaktive polizeiliche Revolution die Mitte der 1990 er Jahre begann hat die innerstädtische Mordrate drastisch gesenkt und Zehntausende von schwarzen Leben gerettet leider ist dieser Rückgang der Kriminalität jetzt in Gefahr wie ich in meinem Buch der Krieg gegen \n"
     ]
    }
   ],
   "source": [
    "!head -n 5 ../data/transcripts/transcripts-all-chunked.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# To train use:\n",
    "`bash train-mlm-xlm.sh &>> train_log.txt`\n",
    "\n",
    "in a separate terminal:\n",
    "`tail -f train_log.txt`\n",
    "\n",
    "This way if you get logged out, or lose the session, you can always pick up the stdout"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# English ngrams\n",
    "The first experiments were done using ngrams generated from english_snippet_graph_matches_100k (and not the above transcripts files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19904\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_2d36f318_9e09_11ec_bc1a_10ddb1cacadb th {\n",
       "          text-align: left;\n",
       "    }#T_2d36f318_9e09_11ec_bc1a_10ddb1cacadbrow0_col0,#T_2d36f318_9e09_11ec_bc1a_10ddb1cacadbrow1_col0,#T_2d36f318_9e09_11ec_bc1a_10ddb1cacadbrow2_col0,#T_2d36f318_9e09_11ec_bc1a_10ddb1cacadbrow3_col0,#T_2d36f318_9e09_11ec_bc1a_10ddb1cacadbrow4_col0{\n",
       "            text-align:  left;\n",
       "        }</style><table id=\"T_2d36f318_9e09_11ec_bc1a_10ddb1cacadb\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >snippet</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_2d36f318_9e09_11ec_bc1a_10ddb1cacadblevel0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_2d36f318_9e09_11ec_bc1a_10ddb1cacadbrow0_col0\" class=\"data row0 col0\" >shots. The people are getting Now. Cover that you're okay. You're not going to, you're not going to get covid. You have these vaccinations, you're not going to get covid-19 have these vaccinations. Guess what? People vaccinated fully vaccinated. People fully vaccinated with boosters people fully vaccinated with boosters and even natural immunity. You're all getting arm across one of the Attorneys General. And there are a number of them around the country have been leading the effort to fight</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_2d36f318_9e09_11ec_bc1a_10ddb1cacadblevel0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "                        <td id=\"T_2d36f318_9e09_11ec_bc1a_10ddb1cacadbrow1_col0\" class=\"data row1 col0\" >It's it's insane. But you know, she wants to close my brother. She wants to close. My sister is so, finally this week. We shoot her. But this is the Deep state that my father's been talking about. For years. They weren't successful in taking down my father in Washington, d.c., Despite the fact that they tried over and over and over again. That's what did they do? They send it to their cronies in New York to try and take him down and it's disgusting. And honestly, I used to have a lot of faith in the legal system in this country. I have no faith in it anymore because of a prosecutor in the United States of America, shouldn't</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_2d36f318_9e09_11ec_bc1a_10ddb1cacadblevel0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "                        <td id=\"T_2d36f318_9e09_11ec_bc1a_10ddb1cacadbrow2_col0\" class=\"data row2 col0\" >going to be honest. When I saw the tape put together and I put one together previously now that I saw some of the other new available video that's come out. I was I was shocked to be honest with you, anything anymore. I'm speaking about politics for what 67 years of this point. And you know, we live to the Russia hoax right? Where the FBI illegally spied on my father's campaign and made up collusion story. Nissan D21. You saw what they did to</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_2d36f318_9e09_11ec_bc1a_10ddb1cacadblevel0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "                        <td id=\"T_2d36f318_9e09_11ec_bc1a_10ddb1cacadbrow3_col0\" class=\"data row3 col0\" >Works through all phases of illness, because it inhibits both viral replication and modulates. The immune response. Of note chloroquine phosphate or hydroxychloroquine identified in April 2020, could actually be a treatment is identified in The Proposal as a SARS Covey to inhibitor. What does all that mean? In the internal documents, the government was passing around. It was showing that</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_2d36f318_9e09_11ec_bc1a_10ddb1cacadblevel0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "                        <td id=\"T_2d36f318_9e09_11ec_bc1a_10ddb1cacadbrow4_col0\" class=\"data row4 col0\" >Free People, which was the freedom to choose Milton Friedman idea that that democracy is voting on the color of your tie, which is a quote, and it was the tenants of this project for privatisation of the public sphere deregulation of the financial biron and everything that would release Capital to be as free as possible and austerity in the public sphere. And, of course, that that was accompanied by Matt criminalization and a divestment, from all of the time. It's an estate that actually help people. And</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fbb9537e160>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "path_to_data = \"data/english_snippet_graph_matches_100k.csv\"\n",
    "df100k = pd.read_csv(path_to_data)\n",
    "# df100k.columns\n",
    "print(len(df100k))\n",
    "with pd.option_context('display.max_colwidth', None):\n",
    "    display(df100k[:5]\n",
    "            .style.set_properties(**{'text-align': 'left'})\n",
    "            .set_table_styles([ dict(selector='th', props=[('text-align', 'left')])]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Diversion - peek at KG entities"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "   \n",
    "SELECT kg_entities.label, kg_entities.risk_level,kg_classifications.label as classification, array_agg(DISTINCT(narratives.label)) as narratives  from kg_entities\n",
    "INNER JOIN kg_classifications on kg_classifications.id = kg_entities.classification_type\n",
    "LEFT JOIN narratives on narratives.id = ANY(kg_entities.narratives)\n",
    "WHERE kg_entities.enabled=true\n",
    "AND kg_entities.risk_level > 2\n",
    "GROUP BY kg_entities.label, kg_entities.risk_level,kg_classifications.label, kg_entities.enabled\n",
    "\n",
    "\n",
    "data/kg-entities.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>risk_level</th>\n",
       "      <th>classification</th>\n",
       "      <th>narratives</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ᛋᛋ</td>\n",
       "      <td>3</td>\n",
       "      <td>Dog Whistle</td>\n",
       "      <td>{\"White Supremacy\"}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>✡👃</td>\n",
       "      <td>4</td>\n",
       "      <td>Dog Whistle</td>\n",
       "      <td>{Antisemitism}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0b@ma</td>\n",
       "      <td>3</td>\n",
       "      <td>Dog Whistle</td>\n",
       "      <td>{Anti-Black,QAnon}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>€0ViD</td>\n",
       "      <td>3</td>\n",
       "      <td>Dog Whistle</td>\n",
       "      <td>{COVID-Denialism}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10 days of darkness</td>\n",
       "      <td>3</td>\n",
       "      <td>Dog Whistle</td>\n",
       "      <td>{QAnon,\"US Election Integrity\"}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3144</th>\n",
       "      <td>黃媒黑記</td>\n",
       "      <td>3</td>\n",
       "      <td>Dog Whistle</td>\n",
       "      <td>{\"HK-Anti-Democracy Protesters\"}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3145</th>\n",
       "      <td>黃獨黑暴</td>\n",
       "      <td>3</td>\n",
       "      <td>Dog Whistle</td>\n",
       "      <td>{\"HK-Anti-Democracy Protesters\"}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3146</th>\n",
       "      <td>黃絲</td>\n",
       "      <td>3</td>\n",
       "      <td>Dog Whistle</td>\n",
       "      <td>{\"HK-Anti-Democracy Protesters\"}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3147</th>\n",
       "      <td>黑學生害死香港</td>\n",
       "      <td>3</td>\n",
       "      <td>Phrase or slogan</td>\n",
       "      <td>{\"HK-Anti-Democracy Protesters\"}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3148</th>\n",
       "      <td>黑暴</td>\n",
       "      <td>3</td>\n",
       "      <td>Dog Whistle</td>\n",
       "      <td>{\"HK-2019 Protests\",\"HK-Anti-Democracy Protest...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3149 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    label  risk_level    classification  \\\n",
       "0                      ᛋᛋ           3       Dog Whistle   \n",
       "1                      ✡👃           4       Dog Whistle   \n",
       "2                   0b@ma           3       Dog Whistle   \n",
       "3                   €0ViD           3       Dog Whistle   \n",
       "4     10 days of darkness           3       Dog Whistle   \n",
       "...                   ...         ...               ...   \n",
       "3144                 黃媒黑記           3       Dog Whistle   \n",
       "3145                 黃獨黑暴           3       Dog Whistle   \n",
       "3146                   黃絲           3       Dog Whistle   \n",
       "3147              黑學生害死香港           3  Phrase or slogan   \n",
       "3148                   黑暴           3       Dog Whistle   \n",
       "\n",
       "                                             narratives  \n",
       "0                                   {\"White Supremacy\"}  \n",
       "1                                        {Antisemitism}  \n",
       "2                                    {Anti-Black,QAnon}  \n",
       "3                                     {COVID-Denialism}  \n",
       "4                       {QAnon,\"US Election Integrity\"}  \n",
       "...                                                 ...  \n",
       "3144                   {\"HK-Anti-Democracy Protesters\"}  \n",
       "3145                   {\"HK-Anti-Democracy Protesters\"}  \n",
       "3146                   {\"HK-Anti-Democracy Protesters\"}  \n",
       "3147                   {\"HK-Anti-Democracy Protesters\"}  \n",
       "3148  {\"HK-2019 Protests\",\"HK-Anti-Democracy Protest...  \n",
       "\n",
       "[3149 rows x 4 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kg_ents = pd.read_csv(\"data/kg-entities.csv\")\n",
    "kg_ents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{Anti-Vaccine}                                                                               542\n",
       "{\"COVID-Pandemic Policies\"}                                                                  239\n",
       "{COVID-Denialism}                                                                            200\n",
       "{Antisemitism}                                                                               164\n",
       "{\"White Supremacy\"}                                                                          128\n",
       "                                                                                            ... \n",
       "{\"German-Far Right\",Misogyny}                                                                  1\n",
       "{\"Global Control Conspiracies\",QAnon,US-Militia}                                               1\n",
       "{Anti-Black,\"German-Far Right\",\"German-Migrants & Refugees\",\"Global-Migrants & Refugees\"}      1\n",
       "{\"COVID-Pandemic Policies\",Islamophobia}                                                       1\n",
       "{\"HK-Anti-Democracy Protesters\",\"HK-National Security Law\"}                                    1\n",
       "Name: narratives, Length: 253, dtype: int64"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kg_ents['narratives'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "# kg_ents['narratives'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting sentence_transformers\n",
      "  Downloading sentence-transformers-2.2.0.tar.gz (79 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.7/79.7 KB\u001b[0m \u001b[31m14.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting transformers<5.0.0,>=4.6.0\n",
      "  Downloading transformers-4.17.0-py3-none-any.whl (3.8 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m99.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: tqdm in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from sentence_transformers) (4.49.0)\n",
      "Requirement already satisfied: torch>=1.6.0 in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from sentence_transformers) (1.10.2)\n",
      "Collecting torchvision\n",
      "  Downloading torchvision-0.12.0-cp37-cp37m-manylinux1_x86_64.whl (21.0 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.0/21.0 MB\u001b[0m \u001b[31m75.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: numpy in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from sentence_transformers) (1.19.2)\n",
      "Requirement already satisfied: scikit-learn in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from sentence_transformers) (0.23.2)\n",
      "Requirement already satisfied: scipy in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from sentence_transformers) (1.5.2)\n",
      "Requirement already satisfied: nltk in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from sentence_transformers) (3.5)\n",
      "Requirement already satisfied: sentencepiece in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from sentence_transformers) (0.1.91)\n",
      "Collecting huggingface-hub\n",
      "  Downloading huggingface_hub-0.4.0-py3-none-any.whl (67 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.0/67.0 KB\u001b[0m \u001b[31m15.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: typing-extensions in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from torch>=1.6.0->sentence_transformers) (3.10.0.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (20.4)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (2022.3.2)\n",
      "Collecting tokenizers!=0.11.3,>=0.11.1\n",
      "  Downloading tokenizers-0.11.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.5 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.5/6.5 MB\u001b[0m \u001b[31m104.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: requests in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (2.27.1)\n",
      "Requirement already satisfied: filelock in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (3.0.12)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (5.4.1)\n",
      "Requirement already satisfied: importlib-metadata in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (2.0.0)\n",
      "Collecting sacremoses\n",
      "  Downloading sacremoses-0.0.47-py2.py3-none-any.whl (895 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m895.2/895.2 KB\u001b[0m \u001b[31m87.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting packaging>=20.0\n",
      "  Downloading packaging-21.3-py3-none-any.whl (40 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.8/40.8 KB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: click in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from nltk->sentence_transformers) (8.0.4)\n",
      "Requirement already satisfied: joblib in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from nltk->sentence_transformers) (1.1.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from scikit-learn->sentence_transformers) (3.1.0)\n",
      "Collecting torch>=1.6.0\n",
      "  Downloading torch-1.11.0-cp37-cp37m-manylinux1_x86_64.whl (750.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m750.6/750.6 MB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pillow!=8.3.*,>=5.3.0 in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from torchvision->sentence_transformers) (9.0.1)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from packaging>=20.0->transformers<5.0.0,>=4.6.0->sentence_transformers) (2.4.7)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from importlib-metadata->transformers<5.0.0,>=4.6.0->sentence_transformers) (3.7.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from requests->transformers<5.0.0,>=4.6.0->sentence_transformers) (1.26.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from requests->transformers<5.0.0,>=4.6.0->sentence_transformers) (2021.10.8)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from requests->transformers<5.0.0,>=4.6.0->sentence_transformers) (3.3)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from requests->transformers<5.0.0,>=4.6.0->sentence_transformers) (2.0.12)\n",
      "Requirement already satisfied: six in /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages (from sacremoses->transformers<5.0.0,>=4.6.0->sentence_transformers) (1.16.0)\n",
      "Building wheels for collected packages: sentence_transformers\n",
      "  Building wheel for sentence_transformers (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for sentence_transformers: filename=sentence_transformers-2.2.0-py3-none-any.whl size=120751 sha256=1d823e575dcf3396611320a0acd8337bcbd3736fb1175b17d9713060978ebc9a\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/83/c0/df/b6873ab7aac3f2465aa9144b6b4c41c4391cfecc027c8b07e7\n",
      "Successfully built sentence_transformers\n",
      "Installing collected packages: tokenizers, torch, packaging, torchvision, huggingface-hub, sacremoses, transformers, sentence_transformers\n",
      "  Attempting uninstall: tokenizers\n",
      "    Found existing installation: tokenizers 0.8.1rc2\n",
      "    Uninstalling tokenizers-0.8.1rc2:\n",
      "      Successfully uninstalled tokenizers-0.8.1rc2\n",
      "  Attempting uninstall: torch\n",
      "    Found existing installation: torch 1.10.2\n",
      "    Uninstalling torch-1.10.2:\n",
      "      Successfully uninstalled torch-1.10.2\n",
      "  Attempting uninstall: packaging\n",
      "    Found existing installation: packaging 20.4\n",
      "    Uninstalling packaging-20.4:\n",
      "      Successfully uninstalled packaging-20.4\n",
      "Successfully installed huggingface-hub-0.4.0 packaging-21.3 sacremoses-0.0.47 sentence_transformers-2.2.0 tokenizers-0.11.6 torch-1.11.0 torchvision-0.12.0 transformers-4.17.0\n"
     ]
    }
   ],
   "source": [
    "!pip install sentence_transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # load model and tokenize a sentence\n",
    "# from sentence_transformers import SentenceTransformer\n",
    "# model = SentenceTransformer(\"../models/xlm-roberta-base\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c990e085951f47a6b8ce99cfaf7f8634",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=615.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28c68e05eb394a52af60bbade000133f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=5069051.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0b50dfb8dc341a197d3ff9e42604066",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=9096718.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a3aa8c180264d79bb341c89c5d07ee4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1115590446.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForMaskedLM\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained('xlm-roberta-base')\n",
    "model = AutoModelForMaskedLM.from_pretrained(\"xlm-roberta-base\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['▁Vac', 'cine', 's', '▁provide', '▁immun', 'ity']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"Vaccines provide immunity\"\n",
    "tokenizer.tokenize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}