Language models_old.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import json\n",
    "import os\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import CountVectorizer as CV\n",
    "import nltk\n",
    "import kenlm\n",
    "import re\n",
    "\n",
    "from scipy.stats import ttest_ind, ttest_rel\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.display import display, HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sfp_data = pd.read_csv(\"../sample_data/sfp_data_v4.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "td_data = pd.read_csv(\"../sample_data/td_data_v4.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "sfp_authors = sfp_data['author'].unique()\n",
    "td_authors = td_data['author'].unique()\n",
    "\n",
    "sfp_posts = sfp_data.loc[sfp_data['subreddit'] == 'SandersForPresident']\n",
    "td_posts = td_data.loc[td_data['subreddit'] == 'The_Donald']\n",
    "\n",
    "sfp_min_times = [min(sfp_posts.loc[sfp_posts['author'] == auth]['created_utc']) for auth in sfp_authors]\n",
    "td_min_times = [min(td_posts.loc[td_posts['author'] == auth]['created_utc']) for auth in td_authors]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4868749, 5)\n",
      "(4003940, 5)\n",
      "2000\n",
      "2000\n",
      "(78037, 5)\n",
      "(55521, 5)\n"
     ]
    }
   ],
   "source": [
    "print(sfp_data.shape)\n",
    "print(td_data.shape)\n",
    "print(len(sfp_authors))\n",
    "print(len(td_authors))\n",
    "print(sfp_posts.shape)\n",
    "print(td_posts.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_overall_time = 1420131600\n",
    "max_overall_time = 1464753600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(131076, 5)\n",
      "(42352, 5)\n"
     ]
    }
   ],
   "source": [
    "sfp_politics = sfp_data.loc[sfp_data['subreddit'] == 'politics']\n",
    "td_politics = td_data.loc[td_data['subreddit'] == 'politics']\n",
    "\n",
    "sfp_politics = sfp_politics.loc[sfp_politics['created_utc'] >= min_overall_time]\n",
    "sfp_politics = sfp_politics.loc[sfp_politics['created_utc'] < max_overall_time]\n",
    "\n",
    "td_politics = td_politics.loc[td_politics['created_utc'] >= min_overall_time]\n",
    "td_politics = td_politics.loc[td_politics['created_utc'] < max_overall_time]\n",
    "\n",
    "print(sfp_politics.shape)\n",
    "print(td_politics.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "sfp_politics_before = pd.DataFrame([], columns = sfp_politics.columns)\n",
    "sfp_politics_after = pd.DataFrame([], columns = sfp_politics.columns)\n",
    "\n",
    "for i in range(len(sfp_authors)):\n",
    "    curr_auth_posts = sfp_politics.loc[sfp_politics['author'] == sfp_authors[i]]\n",
    "    \n",
    "    if len(curr_auth_posts) > 0:\n",
    "        sfp_politics_before = \\\n",
    "sfp_politics_before.append(curr_auth_posts.loc[curr_auth_posts['created_utc'] < sfp_min_times[i]])\n",
    "        \n",
    "        sfp_politics_after = \\\n",
    "sfp_politics_after.append(curr_auth_posts.loc[curr_auth_posts['created_utc'] > sfp_min_times[i]])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "td_politics_before = pd.DataFrame([], columns = td_politics.columns)\n",
    "td_politics_after = pd.DataFrame([], columns = td_politics.columns)\n",
    "\n",
    "for i in range(len(td_authors)):\n",
    "    curr_auth_posts = td_politics.loc[td_politics['author'] == td_authors[i]]\n",
    "    \n",
    "    if len(curr_auth_posts) > 0:\n",
    "        td_politics_before = \\\n",
    "td_politics_before.append(curr_auth_posts.loc[curr_auth_posts['created_utc'] < td_min_times[i]])\n",
    "        \n",
    "        td_politics_after = \\\n",
    "td_politics_after.append(curr_auth_posts.loc[curr_auth_posts['created_utc'] > td_min_times[i]])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "sfp_politics_before_text = sfp_politics_before.loc[sfp_politics_before['body'].notna()]['body'].values\n",
    "sfp_politics_after_text = sfp_politics_after.loc[sfp_politics_after['body'].notna()]['body'].values\n",
    "\n",
    "td_politics_before_text = td_politics_before.loc[td_politics_before['body'].notna()]['body'].values\n",
    "td_politics_after_text = td_politics_after.loc[td_politics_after['body'].notna()]['body'].values\n",
    "\n",
    "sfp_posts_text = sfp_posts.loc[sfp_posts['body'].notna()]['body'].values\n",
    "td_posts_text = td_posts.loc[td_posts['body'].notna()]['body'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_data(data, deleted_str = '[deleted]', removed_str = '[removed]', verbose=True):\n",
    "    if(verbose):\n",
    "        print(\"Number of entries in original dataset: \" + str(len(data)))\n",
    "    data1 = [i for i in data if 'thank you for participating' not in i.lower()]\n",
    "    if(verbose):\n",
    "        print(\"Number of moderator posts removed from dataset: \" + str(len([i for i in data if 'thank you for participating' in i.lower()])))\n",
    "    data1 = [i for i in data1 if i != deleted_str and i != removed_str]\n",
    "    if(verbose):\n",
    "        print(\"Number of user-deleted posts removed from dataset: \" + str(len([i for i in data if i == deleted_str])))\n",
    "        print(\"Number of mod/admin-deleted posts removed from dataset: \" + str(len([i for i in data if i == removed_str])))\n",
    "    \n",
    "        print(\"Number of entries in modified dataset: \" + str(len(data1)))\n",
    "    \n",
    "    # removing hyperlinks\n",
    "        print(\"Number of comments with a hyperlink in modified dataset: \" + str(len([i for i in data1 if \"http\" in i])))\n",
    "    \n",
    "    data1 = [re.sub(r'https?:\\/\\/[\\S]+', ' ', i, flags=re.MULTILINE) for i in data1]\n",
    "    \n",
    "    # common escape sequences\n",
    "    data1 = [re.sub(r'\\n', '', i) for i in data1]\n",
    "    data1 = [re.sub(r'&gt;', '', i) for i in data1]\n",
    "    \n",
    "    # convert to lowercase\n",
    "    data1 = [i.lower() for i in data1]\n",
    "    \n",
    "    # keep user and subreddit tags\n",
    "    if(verbose):\n",
    "        print(\"Number of comments that mention a user in the modified dataset: \" + str(len([i for i in data1 if \"/u/\" in i])))\n",
    "        print(\"Number of comments that mention a subreddit in the modified dataset: \" + str(len([i for i in data1 if \"/r/\" in i])))\n",
    "    \n",
    "    data1 = [re.sub(r'/u/', '_user_', i) for i in data1]\n",
    "    data1 = [re.sub(r'/r/', '_subreddit_', i) for i in data1]\n",
    "    \n",
    "    data_tokenized = [re.findall(r'\\w+', i.lower()) for i in data1]\n",
    "    \n",
    "    if(verbose):\n",
    "        print(\"Number of total tokens: \" + str(sum([len(d) for d in data_tokenized])))\n",
    "    \n",
    "    return [(' '.join(i) + '\\n') for i in data_tokenized]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of entries in original dataset: 28504\n",
      "Number of moderator posts removed from dataset: 0\n",
      "Number of user-deleted posts removed from dataset: 0\n",
      "Number of mod/admin-deleted posts removed from dataset: 0\n",
      "Number of entries in modified dataset: 28504\n",
      "Number of comments with a hyperlink in modified dataset: 1523\n",
      "Number of comments that mention a user in the modified dataset: 79\n",
      "Number of comments that mention a subreddit in the modified dataset: 232\n",
      "Number of total tokens: 1226203\n",
      "\n",
      "Number of entries in original dataset: 102572\n",
      "Number of moderator posts removed from dataset: 52\n",
      "Number of user-deleted posts removed from dataset: 1\n",
      "Number of mod/admin-deleted posts removed from dataset: 0\n",
      "Number of entries in modified dataset: 102519\n",
      "Number of comments with a hyperlink in modified dataset: 6326\n",
      "Number of comments that mention a user in the modified dataset: 201\n",
      "Number of comments that mention a subreddit in the modified dataset: 731\n",
      "Number of total tokens: 4335851\n",
      "\n",
      "Number of entries in original dataset: 23854\n",
      "Number of moderator posts removed from dataset: 0\n",
      "Number of user-deleted posts removed from dataset: 0\n",
      "Number of mod/admin-deleted posts removed from dataset: 0\n",
      "Number of entries in modified dataset: 23854\n",
      "Number of comments with a hyperlink in modified dataset: 828\n",
      "Number of comments that mention a user in the modified dataset: 49\n",
      "Number of comments that mention a subreddit in the modified dataset: 208\n",
      "Number of total tokens: 940808\n",
      "\n",
      "Number of entries in original dataset: 18498\n",
      "Number of moderator posts removed from dataset: 1\n",
      "Number of user-deleted posts removed from dataset: 0\n",
      "Number of mod/admin-deleted posts removed from dataset: 0\n",
      "Number of entries in modified dataset: 18497\n",
      "Number of comments with a hyperlink in modified dataset: 750\n",
      "Number of comments that mention a user in the modified dataset: 32\n",
      "Number of comments that mention a subreddit in the modified dataset: 187\n",
      "Number of total tokens: 570210\n",
      "\n",
      "Number of entries in original dataset: 78036\n",
      "Number of moderator posts removed from dataset: 0\n",
      "Number of user-deleted posts removed from dataset: 0\n",
      "Number of mod/admin-deleted posts removed from dataset: 0\n",
      "Number of entries in modified dataset: 78036\n",
      "Number of comments with a hyperlink in modified dataset: 5448\n",
      "Number of comments that mention a user in the modified dataset: 238\n",
      "Number of comments that mention a subreddit in the modified dataset: 449\n",
      "Number of total tokens: 2751940\n",
      "\n",
      "Number of entries in original dataset: 55519\n",
      "Number of moderator posts removed from dataset: 1\n",
      "Number of user-deleted posts removed from dataset: 2\n",
      "Number of mod/admin-deleted posts removed from dataset: 4\n",
      "Number of entries in modified dataset: 55512\n",
      "Number of comments with a hyperlink in modified dataset: 2813\n",
      "Number of comments that mention a user in the modified dataset: 117\n",
      "Number of comments that mention a subreddit in the modified dataset: 716\n",
      "Number of total tokens: 1310787\n",
      "\n"
     ]
    }
   ],
   "source": [
    "text_arrs = [sfp_politics_before_text,\n",
    "             sfp_politics_after_text,\n",
    "             td_politics_before_text,\n",
    "             td_politics_after_text,\n",
    "             sfp_posts_text,\n",
    "             td_posts_text]\n",
    "\n",
    "test_arrs = []\n",
    "\n",
    "for i in range(len(text_arrs)):\n",
    "    lm_test_arr = tokenize_data(text_arrs[i])\n",
    "    print()\n",
    "    test_arrs.append(lm_test_arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28504\n",
      "['what state do you live in\\n', 'awesome\\n', 'a link to the actual bill\\n', 'can you provide a source proving that planned parenthood is using donations to some kind of real benefit how are they getting around the law are they not just donating to the scientific community from the article it turns out nucatola wasn t discussing the illegal black market sale of fetal organs but instead the perfectly legal donation of the organs to biomedical research laboratories that use the organs to help save lives the price range of the organs described by nucatola is in reality the reimbursed expenses accrued in the delivery of the specimens\\n', 'which was a total misrepresentation of the question he brought up reassignment himself and then went on to describe what the military does standard garbage response\\n', 'what if hillary gets the nomination despite the popular vote being solidly behind sanders if the democratic base is convinced the party went against the will of the people in favor of backroom deals won t sanders reconsider running 3rd party rather than giving hillary a free pass\\n', 'people were saying this same stupid shit when bush was president and clinton before him in reality all presidents regardless of their political affiliations are playing the same game and are part of the same team there is no benefit for the people they serve wealthy elite to do something like this in fact this kind of theory is only perpetuating the partisan bickering which is what those in power want it forces people to take sides and stop looking at the big picture\\n', 'fucking yawn does anybody really think putting another clinton or bush in the white house is going to create any real change in america both families are proven servants to their corporate overlords not the american people\\n', 'i really like bernie sanders but unfortunately he has no chance of winning if he runs at the end of the day the money isn t on his side and american voters don t give enough of a shit to get off their lazy asses and vote also he doesn t look very presidential which unfortunately in america actually makes a difference\\n', 'fucking scumbags\\n']\n",
      "\n",
      "\n",
      "\n",
      "102519\n",
      "['there s a lot involved including exit polls and demographics precinct math\\n', 'halfway down this page is the table that matters the candidates need to hit these delegate marks to be on pace for the nomination\\n', 'edited to say the candidates i posted hastily\\n', 'i do not believe those are exit polls from today they may be for early voting in that state also that content appears to be down now maybe the made a mistake edit yup these are placeholder numbers it s way too early for exit numbers\\n', 'i was there today flew out from los angeles and was among those arrested on the steps of the capitol if you have questions let me know most questions about the movement itself can be found at democracyspring org including specific legislation proposed to combat the influence of big money interests in politics and voter suppression issues here s a photo i took from the sit in on the steps you can see the rally on the other side of the police line for the most part the capitol city police were friendly i spent some time during processing talking to one of them about the warriors for a while here are a couple of short videos of the event one from the march as we reached the capitol and one from the steps right as we were about to begin the sit in there was a 3 hour training session for those participating in the sit in so many of us were well prepared and well informed many people had participated in the arrests throughout the week and others had lots of experience in protests in years past the organization takes nonviolent direct action very seriously members include noam chomsky and lawrence lessig as well as celebrities like mark ruffalo those who chose to sit in on the steps of the capitol in violation of the law and be arrested were also fined 50 resulting in 45 000 in fines some people did go to jail and others were processed on site see details in comments below those who chose not to sit in the majority protested in the designated protest area the protests earlier in the week had coverage by better news organizations if you want to participate there are two events in dc upcoming by the sister organization democracy awakening democracy awakens this sunday and mondaysunday rally for democracy 1 00 pm west lawn of the capitol buildingcongress of conscience day of action kick off rally 9 00 am columbus circle you can also find local democracy spring hubs to participate in your area\\n', 'it is a bit unfortunate that we require celebrity endorsement to bring attention to it cnn has also reported on it though they only covered the first day there has been a lack of mainstream media coverage given that this is shaping up to be a record for number of nonviolent protest arrests at the capitol\\n', 'yes if you look at the proposed legislation one includes an amendment with this aim in mind of course that s very difficult and there are other approaches it s a challenge and an important step is to make persuade congress to accept that challenge\\n', 'on monday the capitol city police jails were at capacity due to the sit in arrests which prompted remote processing towards the end of this week of sit ins the capitol city police ended up processing protesters on site with a kind of remote holding area they did not take us to jail today at least no one i saw there may have been some that were taken yesterday and in days prior some people were taken to jail and some were processed on site\\n', 'to be clear there have been thousands of democracy spring protesters over the course of the week at the capitol of these over 900 people chose to sit in on the steps of the capitol and choose to be arrested as a form of nonviolent civil disobedience we knew we would be arrested and the capitol city police were prepared for us the actual citation we received states crowding obstructing or incommoding d c code 22 1307 because we were sitting on the steps of the capitol the goal was awareness many more people did not elect to be arrested but instead stayed in the permitted protest area\\n', 'awesome glad you guys were out there\\n']\n",
      "\n",
      "\n",
      "\n",
      "23854\n",
      "['really it s more stupid by the american public to fall for that in other countries politicians have done similarly stupid things but mainly in america are politicians judged so much on their charisma\\n', 'that s not how data analysis works the way it works is a scientist writes a computer program which analyses it all in one go you need a couple of geniuses not 10000 monkeys\\n', 'that could certainly be the case but i would not call that crowd sourcing\\n', 'it s not but i would not call linux crowdsourcing either perhaps my idea of crowdsourcing is wrong but as i understand the term that s about getting a whole bunch of people to perform a task collectively and not about picking a handful of domain experts out of a crowd i probably misunderstood you initially i thought by pouring over data for 3 4 hours a week you meant manually looking at a whole bunch of data values and trying to find something in it and then getting a whole crowd of people to do the same\\n', 'it is much easier to just blame the liberal media instead of reading the first sentence of the article and get the link to the study typical\\n', 'if the government would open the nics to the public at a cost of under 5 then there might be some traction with the idea and at the same time you can get rid of all of the states registries and remove any states waiting period\\n', 'then congress should just exert their political pressure to have some other obscure and completely not related agency department to have a third party conduct this research like maybe federal railroad administration or veteran affairs or something because that makes sense you ve heard of the atf right\\n', 'could the same be said for hildog\\n', 'anybody know what ja rule thinks about clinton vs sanders\\n', 'if a protester starts demonstrating in the area around you please do not touch or harm the protester this is a peaceful rally and one of the protesters arrested for shutting down the freeway in arizona in 2011 she was awarded the soros justice fellowship which grants between 58 000 to 110 000 to individuals to undertake projects that advance reform catalyze change on a range of issues facing the criminal justice system in the us and spur debate trump has since backtracked on his paying the legal fees comment my guess is because if he does that directly it could be problematic but if somebody started a non profit he could donate to that was setup to disrupt other campaigns i m sure he d consider that\\n']\n",
      "\n",
      "\n",
      "\n",
      "18497\n",
      "['fuck i guess i m still in the running for the dnc candidate then too\\n', 'innocent until proven guilty what would stop hrc or her camp or any random joe for that matter from making unfounded allegations against bernie sanders or any political opponent that would prompt an fbi investigation immediately making them ineligible for the nomination\\n', 'i never said that i m saying you don t make somebody ineligible for the dnc nomination based on allegations you are trying to put words in my mouth and are being disingenuous and you know it\\n', 'i m saying you don t make somebody ineligible for the dnc nomination based on allegations you are trying to put words in my mouth and are being disingenuous and you know it\\n', 'believe it or not it is a thing in a working democracy\\n', 'you are coocooo for coco puffs\\n', 'you are living in fantasy land and in a few weeks gunna get dragged screaming and kicking back to reality and i can t wait\\n', 'i feel like you re the only person to reply to me that has understood what i was saying if allegations disqualify a candidate then allegations will become a prominent tool especially with campaign finance laws it would be quite trivial to prompt an investigation add a couple depositions and now it s gunna take a year\\n', 'remindme july 25 2016i wanna rub it your face when it happens\\n', 'i knew reddit would get around to blaming republicans for fuckups in a democratic primary hilarious\\n']\n",
      "\n",
      "\n",
      "\n",
      "78036\n",
      "['i believe warren is more likely to see the war machine as an acceptable necessity due to her stance on fighting terror and backing israel i d prefer to see a sanders grayson run myself\\n', 'alan grayson he s genuinely progressive and outspoken yet he s managed to find ways to get libertarians and republicans to work with him on common ground issues i could see him being able to publicly say the things a president probably shouldn t and still be able to sway many conservatives to back sanders on constitutional and privacy issues\\n', 'bill deblasio\\n', 'actually frank jackson mayor of cleveland might help as an old white guy i m not sure that sanders would have that much appeal to african americans frank jackson is black locally popular and from a swing state\\n', 'as someone who worked on sherrod brown s campaign i can tell you that i love his record and think he has the washington experience that is necessary for the job but he is not very inspirational he has this raspy voice that sounds like he s been chain smoking since he was 12 years old and is not particularly good at giving speeches this doesn t make him a bad choice for president however it does wear down severely on his electability\\n', 'when did everyone forget about the internet spying and start focusing only on the phone spying\\n', 'i m sure hillary will echo all the progressive talking points during her campaign just as obama did but as we have seen with obama her campaign identity will vanish as soon as she s in office\\n', 'the longest of long shots i really dont think so\\n', 'i d rather he run as an independent the best decision he ever made was leaving the democratic party\\n', 'technically any candidate with 15 support in five national polls is supposedly allowed in the debates i ll admit the commission on presidential debates is definitely biased against allowing third parties and i wouldn t be surprised to see them change the rules to exclude a third party candidate who qualified it really doesn t matter though the internet has made the sham debates and mainstream media coverage less meaningful a smart candidate can gain support through other means any candidate who will bring real positive change to this country is going the have to do this because they will not receive the support of the establishment\\n']\n",
      "\n",
      "\n",
      "\n",
      "55512\n",
      "['keeping it classy\\n', 'look at that picture no shortage of minorities i think bernie sanders should be taking some cues from trump sanders attracts fairly lilly white crowds\\n', 'probably because he s pro illegal immigration i can t think of a better explanation\\n', 'he s currently polling at twice the next best candidate in iowa if the mccain thing didn t sink trump what do you think it ll take let s see how february works out for him he s in the top tier\\n', 'he s been the clear leader in every poll i ve seen so far the establishment hates his guts the base knows he isn t a conservative and he s got this very passionate support anyways i don t see trump as a flash in the pan i see him as being a pain in the ass for the rest of the gop until the primaries are well under way and the bottom tier starts dropping out\\n', 'the sjwers assume that latinos will stick together in unison with their illegal bretheren they re obviously wrong\\n', 'if they re taking notes the party will come roaring back stronger than ever ted cruz with more experience will give the trump supporters what they want christie is sort of like this as well but he s damaged goods the party needs guys who stand for something in order to win even if a pushover like jeb bush somehow makes it into the white house the conservatives lose people need to understand this\\n', 'perfect for the outsider candidate\\n', 'so erickson called a sc justice a goat fucking child molester but trump gets disinvited from the red state gathering for using the word whereever it s clear that erickson was just waiting for a good excuse to dump trump we all know trump isn t a true conservative and he isn t going to take orders from any big donors or anyone in the establishment these guys are afraid of trump\\n', 'this is a man who gets it\\n']\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(test_arrs)):\n",
    "    print(len(test_arrs[i]))\n",
    "    print(test_arrs[i][:10])\n",
    "    print(\"\\n\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28504\n",
      "102519\n",
      "23854\n",
      "18497\n",
      "78036\n",
      "55512\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(test_arrs)):\n",
    "    print(len(test_arrs[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "filenames = ['politics_train.arpa',\n",
    "             'sfp_politics_before.arpa', \n",
    "             'sfp_politics_after.arpa',\n",
    "             'td_politics_before.arpa',\n",
    "             'td_politics_after.arpa',\n",
    "             'sfp_posts.arpa',\n",
    "             'td_posts.arpa']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2-gram model\n",
      "2-gram model\n",
      "2-gram model\n",
      "2-gram model\n",
      "2-gram model\n",
      "2-gram model\n",
      "2-gram model\n"
     ]
    }
   ],
   "source": [
    "language_models = []\n",
    "\n",
    "for i in range(len(filenames)):\n",
    "    curr_lm = os.path.join(os.path.dirname('Language models.ipynb'), '..', 'language_models', filenames[i])\n",
    "    curr_model = kenlm.LanguageModel(curr_lm)\n",
    "    print('{0}-gram model'.format(curr_model.order))\n",
    "    language_models.append(curr_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bootstrap(text1, text2, scoring_func, num_iters = 1000, sample_size = 1000):\n",
    "    lower_size = min(len(text1), len(text2))\n",
    "    text1_new = np.random.choice(text1, lower_size, replace=False)\n",
    "    text2_new = np.random.choice(text2, lower_size, replace=False)\n",
    "    \n",
    "    scores_all = [scoring_func(s) for s in (text1 + text2)]\n",
    "    bootstrap1 = []\n",
    "    bootstrap2 = []\n",
    "    random.seed(10)\n",
    "    \n",
    "    for i in range(num_iters):\n",
    "        scores1 = random.sample(scores_all, sample_size)\n",
    "        scores2 = random.sample(scores_all, sample_size)\n",
    "        \n",
    "        bootstrap1.append(sum(scores1)/sample_size)\n",
    "        bootstrap2.append(sum(scores2)/sample_size)\n",
    "        \n",
    "    return (bootstrap1, bootstrap2)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_values(text1, text2, scoring_func, num_iters = 1000, sample_size = 1000):\n",
    "    random.seed(10)\n",
    "    text1_sample = random.sample(text1, sample_size)\n",
    "    text2_sample = random.sample(text2, sample_size)\n",
    "    scores1 = [scoring_func(s) for s in text1_sample]\n",
    "    scores2 = [scoring_func(s) for s in text2_sample]\n",
    "\n",
    "    mean1 = sum(scores1)/len(scores1)\n",
    "    mean2 = sum(scores2)/len(scores2)\n",
    "    print(mean1)\n",
    "    print(mean2)\n",
    "    print()\n",
    "    \n",
    "    bootstrap1, bootstrap2 = bootstrap(text1, text2, scoring_func, num_iters)\n",
    "    model_diff = mean1 - mean2\n",
    "    print(model_diff)\n",
    "    print()\n",
    "    \n",
    "    bootstrap_diffs = [bootstrap1[i] - bootstrap2[i] for i in range(len(bootstrap1))]\n",
    "    print(sum([i > model_diff for i in bootstrap_diffs])/len(bootstrap_diffs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_values_paired(text1, text2, scoring_func):\n",
    "    random.seed(10)\n",
    "    text1_sample = random.sample(text1, sample_size)\n",
    "    text2_sample = random.sample(text2, sample_size)\n",
    "    scores1 = [scoring_func(s) for s in text1_sample]\n",
    "    scores2 = [scoring_func(s) for s in text2_sample]\n",
    "\n",
    "    mean1 = sum(scores1)/len(scores1)\n",
    "    mean2 = sum(scores2)/len(scores2)\n",
    "    print(mean1)\n",
    "    print(mean2)\n",
    "    print()\n",
    "    \n",
    "    \n",
    "    bootstrap1, bootstrap2 = bootstrap(text1, text2, scoring_func, num_iters)\n",
    "    model_diff = mean1 - mean2\n",
    "    print(model_diff)\n",
    "    print()\n",
    "    \n",
    "    bootstrap_diffs = [bootstrap1[i] - bootstrap2[i] for i in range(len(bootstrap1))]\n",
    "    print(sum([i > model_diff for i in bootstrap_diffs])/len(bootstrap_diffs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_values_ind(text1, text2, scoring_func):\n",
    "    scores1 = [scoring_func(s) for s in text1]\n",
    "    scores2 = [scoring_func(s) for s in text2]\n",
    "\n",
    "    mean1 = sum(scores1)/len(scores1)\n",
    "    mean2 = sum(scores2)/len(scores2)\n",
    "    print(mean1)\n",
    "    print(mean2)\n",
    "    print()\n",
    "    \n",
    "    print(ttest_ind(scores1, scores2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "466.0915190667158\n",
      "464.0644101356655\n",
      "\n",
      "2.027108931050293\n",
      "\n",
      "0.512\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, td_politics_before, sfp_politics_before\n",
    "calculate_values(test_arrs[0], test_arrs[2], language_models[1].perplexity, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "415.26142824930616\n",
      "390.3400073401792\n",
      "\n",
      "24.921420909126937\n",
      "\n",
      "0.438\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, td_politics_before, td_politics_before\n",
    "calculate_values(test_arrs[0], test_arrs[2], language_models[3].perplexity, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "348.7865394483283\n",
      "516.1244845746966\n",
      "\n",
      "-167.33794512636825\n",
      "\n",
      "0.877\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_after, td_politics_after, sfp_politics_after\n",
    "calculate_values(test_arrs[1], test_arrs[3], language_models[2].perplexity, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "352.5069705611644\n",
      "414.18445782481314\n",
      "\n",
      "-61.67748726364874\n",
      "\n",
      "0.82\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_after, td_politics_after, td_politics_after\n",
    "calculate_values(test_arrs[1], test_arrs[3], language_models[4].perplexity, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "452.27702955300754\n",
      "442.24635122921\n",
      "\n",
      "10.030678323797531\n",
      "\n",
      "0.478\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, td_politics_before, sfp_posts\n",
    "calculate_values(test_arrs[0], test_arrs[2], language_models[5].perplexity, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "411.0995839632289\n",
      "390.2742375488254\n",
      "\n",
      "20.825346414403498\n",
      "\n",
      "0.376\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, td_politics_before, td_posts\n",
    "calculate_values(test_arrs[0], test_arrs[2], language_models[6].perplexity, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "361.5910337341523\n",
      "526.1723508792212\n",
      "\n",
      "-164.5813171450689\n",
      "\n",
      "0.92\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_after, td_politics_after, sfp_posts\n",
    "calculate_values(test_arrs[1], test_arrs[3], language_models[5].perplexity, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "357.7513617089054\n",
      "376.5504233294564\n",
      "\n",
      "-18.799061620551015\n",
      "\n",
      "0.607\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_after, td_politics_after, td_posts\n",
    "calculate_values(test_arrs[1], test_arrs[3], language_models[6].perplexity, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "386.99331096744527\n",
      "395.1014610810287\n",
      "\n",
      "-8.108150113583406\n",
      "\n",
      "0.531\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, td_politics_before, politics\n",
    "calculate_values(test_arrs[0], test_arrs[2], language_models[0].perplexity, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "349.8563032214515\n",
      "439.71758078008185\n",
      "\n",
      "-89.86127755863032\n",
      "\n",
      "0.93\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_after, td_politics_after, politics\n",
    "calculate_values(test_arrs[1], test_arrs[3], language_models[0].perplexity, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "372.3425881771748\n",
      "1437.7450103397887\n",
      "\n",
      "-1065.402422162614\n",
      "\n",
      "0.998\n"
     ]
    }
   ],
   "source": [
    "# sfp_posts, td_posts, sfp_posts\n",
    "calculate_values(test_arrs[4], test_arrs[5], language_models[5].perplexity, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "383.91765532531605\n",
      "658.0044753733365\n",
      "\n",
      "-274.08682004802046\n",
      "\n",
      "0.963\n"
     ]
    }
   ],
   "source": [
    "# sfp_posts, td_posts, td_posts\n",
    "calculate_values(test_arrs[4], test_arrs[5], language_models[6].perplexity, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "654.1731912362627\n",
      "437.5470999445293\n",
      "\n",
      "216.62609129173342\n",
      "\n",
      "0.069\n"
     ]
    }
   ],
   "source": [
    "# td_posts, sfp_posts, td_posts\n",
    "calculate_values(test_arrs[5], test_arrs[4], language_models[6].perplexity, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "393.96497182012\n",
      "987.3871161287846\n",
      "\n",
      "-593.4221443086645\n",
      "\n",
      "0.999\n"
     ]
    }
   ],
   "source": [
    "# sfp_posts, td_posts, politics\n",
    "calculate_values(test_arrs[4], test_arrs[5], language_models[0].perplexity, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "386.99331096744527\n",
      "357.0470253137992\n",
      "\n",
      "29.946285653646044\n",
      "\n",
      "0.321\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, sfp_politics_after, politics\n",
    "calculate_values(test_arrs[0], test_arrs[1], language_models[0].perplexity, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "380.8497134050213\n",
      "452.1804452783587\n",
      "\n",
      "-71.33073187333741\n",
      "\n",
      "0.849\n"
     ]
    }
   ],
   "source": [
    "# td_politics_before, td_politics_after, politics\n",
    "calculate_values(test_arrs[2], test_arrs[3], language_models[0].perplexity, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Same but with cross entropy instead of perplexity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-40.99284445625543\n",
      "-38.161164723038674\n",
      "\n",
      "-2.8316797332167596\n",
      "\n",
      "0.975\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, td_politics_before, sfp_politics_before\n",
    "calculate_values(test_arrs[0], test_arrs[2], language_models[1].score, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-41.45484392162562\n",
      "-38.48047024730444\n",
      "\n",
      "-2.9743736743211784\n",
      "\n",
      "0.98\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, td_politics_before, td_politics_before\n",
    "calculate_values(test_arrs[0], test_arrs[2], language_models[3].score, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-38.13838319438696\n",
      "-33.79882725139856\n",
      "\n",
      "-4.339555942988397\n",
      "\n",
      "0.998\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_after, td_politics_after, sfp_politics_after\n",
    "calculate_values(test_arrs[1], test_arrs[3], language_models[2].score, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-39.60858447471857\n",
      "-34.6449614354372\n",
      "\n",
      "-4.963623039281366\n",
      "\n",
      "0.999\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_after, td_politics_after, td_politics_after\n",
    "calculate_values(test_arrs[1], test_arrs[3], language_models[4].score, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-42.761745394754406\n",
      "-39.87905867060423\n",
      "\n",
      "-2.882686724150176\n",
      "\n",
      "0.971\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, td_politics_before, sfp_posts\n",
    "calculate_values(test_arrs[0], test_arrs[2], language_models[5].score, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-43.46981461167336\n",
      "-40.32892970715761\n",
      "\n",
      "-3.140884904515744\n",
      "\n",
      "0.981\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, td_politics_before, td_posts\n",
    "calculate_values(test_arrs[0], test_arrs[2], language_models[6].score, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-39.028087789463996\n",
      "-34.663414007985594\n",
      "\n",
      "-4.364673781478402\n",
      "\n",
      "0.998\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_after, td_politics_after, sfp_posts\n",
    "calculate_values(test_arrs[1], test_arrs[3], language_models[5].score, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-40.23067969937325\n",
      "-34.85549478787184\n",
      "\n",
      "-5.375184911501407\n",
      "\n",
      "0.999\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_after, td_politics_after, td_posts\n",
    "calculate_values(test_arrs[1], test_arrs[3], language_models[6].score, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-43.261180846309664\n",
      "-40.2992213262558\n",
      "\n",
      "-2.961959520053867\n",
      "\n",
      "0.973\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, td_politics_before, politics\n",
    "calculate_values(test_arrs[0], test_arrs[2], language_models[0].score, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-39.98813622492552\n",
      "-35.2850863899827\n",
      "\n",
      "-4.703049834942817\n",
      "\n",
      "0.999\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_after, td_politics_after, politics\n",
    "calculate_values(test_arrs[1], test_arrs[3], language_models[0].score, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-36.33988372894525\n",
      "-33.53964806175232\n",
      "\n",
      "-2.8002356671929363\n",
      "\n",
      "0.91\n"
     ]
    }
   ],
   "source": [
    "# sfp_posts, td_posts, sfp_posts\n",
    "calculate_values(test_arrs[4], test_arrs[5], language_models[5].score, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-37.99267998012304\n",
      "-31.907608811330796\n",
      "\n",
      "-6.085071168792247\n",
      "\n",
      "0.988\n"
     ]
    }
   ],
   "source": [
    "# sfp_posts, td_posts, td_posts\n",
    "calculate_values(test_arrs[4], test_arrs[5], language_models[6].score, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-32.96674963059425\n",
      "-37.55648960086107\n",
      "\n",
      "4.58973997026682\n",
      "\n",
      "0.02\n"
     ]
    }
   ],
   "source": [
    "# td_posts, sfp_posts, td_posts\n",
    "calculate_values(test_arrs[5], test_arrs[4], language_models[6].score, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-38.140959322738645\n",
      "-34.11710799072981\n",
      "\n",
      "-4.023851332008839\n",
      "\n",
      "0.963\n"
     ]
    }
   ],
   "source": [
    "# sfp_posts, td_posts, politics\n",
    "calculate_values(test_arrs[4], test_arrs[5], language_models[0].score, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-43.261180846309664\n",
      "-40.16015071448088\n",
      "\n",
      "-3.101030131828786\n",
      "\n",
      "0.981\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, sfp_politics_after, politics\n",
    "calculate_values(test_arrs[0], test_arrs[1], language_models[0].score, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-40.26308662309646\n",
      "-35.172616477310655\n",
      "\n",
      "-5.090470145785808\n",
      "\n",
      "0.999\n"
     ]
    }
   ],
   "source": [
    "# td_politics_before, td_politics_after, politics\n",
    "calculate_values(test_arrs[2], test_arrs[3], language_models[0].score, num_iters=1000, sample_size = 20000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Almost all of these are p < 0.05, but the bootstrapping method seems suspect, so I should use something better. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use a t-test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-120.963218138956\n",
      "-110.61032555850322\n",
      "\n",
      "Ttest_indResult(statistic=-6.732302343462776, pvalue=1.6871667718307063e-11)\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, td_politics_before, sfp_politics_before\n",
    "calculate_values_ind(test_arrs[0], test_arrs[2], language_models[1].score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-121.42584818385806\n",
      "-110.8403906114565\n",
      "\n",
      "Ttest_indResult(statistic=-6.838922919323715, pvalue=8.066378947210291e-12)\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, td_politics_before, td_politics_before\n",
    "calculate_values_ind(test_arrs[0], test_arrs[2], language_models[3].score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-117.70578388786899\n",
      "-86.75577316455869\n",
      "\n",
      "Ttest_indResult(statistic=-24.07462982723636, pvalue=9.230155122576456e-128)\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_after, td_politics_after, sfp_politics_after\n",
    "calculate_values_ind(test_arrs[1], test_arrs[3], language_models[2].score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-120.69819819193371\n",
      "-87.87735906627917\n",
      "\n",
      "Ttest_indResult(statistic=-24.849185357163726, pvalue=5.800221922592684e-136)\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_after, td_politics_after, td_politics_after\n",
    "calculate_values_ind(test_arrs[1], test_arrs[3], language_models[4].score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-122.944396003466\n",
      "-113.06373171211838\n",
      "\n",
      "Ttest_indResult(statistic=-6.210748957124003, pvalue=5.312781221067689e-10)\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, td_politics_before, sfp_posts\n",
    "calculate_values_ind(test_arrs[0], test_arrs[2], language_models[5].score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-124.33040786749721\n",
      "-113.49665718767463\n",
      "\n",
      "Ttest_indResult(statistic=-6.729717322636551, pvalue=1.717386727537896e-11)\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, td_politics_before, td_posts\n",
    "calculate_values_ind(test_arrs[0], test_arrs[2], language_models[6].score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-119.72738866990963\n",
      "-88.33224752282248\n",
      "\n",
      "Ttest_indResult(statistic=-23.88363855931886, pvalue=8.876011556885794e-126)\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_after, td_politics_after, sfp_posts\n",
    "calculate_values_ind(test_arrs[1], test_arrs[3], language_models[5].score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-122.24164702453253\n",
      "-88.22415055580034\n",
      "\n",
      "Ttest_indResult(statistic=-25.310515175826023, pvalue=5.710629924179657e-141)\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_after, td_politics_after, td_posts\n",
    "calculate_values_ind(test_arrs[1], test_arrs[3], language_models[6].score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-124.30374640226364\n",
      "-113.96454048976321\n",
      "\n",
      "Ttest_indResult(statistic=-6.480470301748532, pvalue=9.224664970309833e-11)\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, td_politics_before, politics\n",
    "calculate_values_ind(test_arrs[0], test_arrs[2], language_models[0].score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-121.96106240087934\n",
      "-89.50404411971637\n",
      "\n",
      "Ttest_indResult(statistic=-24.448181531396976, pvalue=1.1006954582946825e-131)\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_after, td_politics_after, politics\n",
    "calculate_values_ind(test_arrs[1], test_arrs[3], language_models[0].score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-97.99008184064631\n",
      "-71.35517761280909\n",
      "\n",
      "Ttest_indResult(statistic=-33.507022626772184, pvalue=3.996217477103932e-245)\n"
     ]
    }
   ],
   "source": [
    "# sfp_posts, td_posts, sfp_posts\n",
    "calculate_values_ind(test_arrs[4], test_arrs[5], language_models[5].score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-101.066193753447\n",
      "-68.05382477948535\n",
      "\n",
      "Ttest_indResult(statistic=-42.33169226761963, pvalue=0.0)\n"
     ]
    }
   ],
   "source": [
    "# sfp_posts, td_posts, td_posts\n",
    "calculate_values_ind(test_arrs[4], test_arrs[5], language_models[6].score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-68.05382477948535\n",
      "-101.066193753447\n",
      "\n",
      "Ttest_indResult(statistic=42.33169226761963, pvalue=0.0)\n"
     ]
    }
   ],
   "source": [
    "# td_posts, sfp_posts, td_posts\n",
    "calculate_values_ind(test_arrs[5], test_arrs[4], language_models[6].score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-101.69964701879812\n",
      "-71.97838298174779\n",
      "\n",
      "Ttest_indResult(statistic=-37.22797900401066, pvalue=8.603074743452662e-302)\n"
     ]
    }
   ],
   "source": [
    "# sfp_posts, td_posts, politics\n",
    "calculate_values_ind(test_arrs[4], test_arrs[5], language_models[0].score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-124.30374640226364\n",
      "-121.96106240087934\n",
      "\n",
      "Ttest_indResult(statistic=-2.0451283099572826, pvalue=0.04084420323411929)\n"
     ]
    }
   ],
   "source": [
    "# sfp_politics_before, sfp_politics_after, politics\n",
    "calculate_values_ind(test_arrs[0], test_arrs[1], language_models[0].score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-113.96454048976321\n",
      "-89.50404411971637\n",
      "\n",
      "Ttest_indResult(statistic=-14.599797337934946, pvalue=3.689276439978331e-48)\n"
     ]
    }
   ],
   "source": [
    "# td_politics_before, td_politics_after, politics\n",
    "calculate_values_ind(test_arrs[2], test_arrs[3], language_models[0].score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-126.20280684804916\n",
      "-116.40277046489716\n",
      "\n",
      "Ttest_indResult(statistic=-1.2691939738407458, pvalue=0.2045197137905985)\n"
     ]
    }
   ],
   "source": [
    "# sanity check\n",
    "calculate_values_ind(random.sample(test_arrs[0], 1000), random.sample(test_arrs[0], 1000), language_models[0].score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Average over users first"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}