CommonWordsbyEmoji-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from IPython.core.display import HTML, display\n",
    "HTML(\"<style>.container { width:100% !important; }</style>\")\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "matplotlib.rcParams['savefig.dpi'] = 2 * matplotlib.rcParams['savefig.dpi']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from Data.sentiment_dict import positive, negative\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "pd.options.display.max_colwidth=250"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# separate tweet into parts with emojis of interest and the rest\n",
    "def separate_emoji(tweet, emojis):\n",
    "    emojis = set(tweet).intersection(emojis)\n",
    "    text = ''\n",
    "    for i in tweet:\n",
    "        if i not in emojis:\n",
    "            text += i\n",
    "    return text, dict.fromkeys(emojis,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# specify the location of the tweets\n",
    "negtweet_file ='./Data/negtweets.txt'\n",
    "postweet_file ='./Data/postweets.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# initial dataframes, one for tweets, one for emoji of interest\n",
    "tweet_df = []\n",
    "emoji_df = []\n",
    "# load all the tweets from positive file\n",
    "with open(postweet_file,'r') as f:\n",
    "    for line in f:\n",
    "        tweet = line.decode('utf-8')\n",
    "        emojis = positive+negative\n",
    "        text_i, emojis_i =separate_emoji(tweet, emojis)\n",
    "        emoji_df.append(emojis_i) \n",
    "        tweet_df.append(text_i)\n",
    "# load all the tweets from negative file\n",
    "with open(negtweet_file,'r') as f:\n",
    "    for line in f:\n",
    "        tweet = line.decode('utf-8')\n",
    "        emojis = positive+negative\n",
    "        text_i, emojis_i =separate_emoji(tweet, emojis)\n",
    "        emoji_df.append(emojis_i) \n",
    "        tweet_df.append(text_i)\n",
    "\n",
    "# convert data into dataframe\n",
    "tweet_df = pd.DataFrame(tweet_df, columns=['tweet'])\n",
    "DV=DictVectorizer()\n",
    "emoji_df=DV.fit_transform(emoji_df)\n",
    "emoji_list = {}\n",
    "for i in DV.feature_names_:\n",
    "    emoji_list[i.encode('raw_unicode_escape')] = i\n",
    "emoji_list=pd.DataFrame(emoji_list, index=[0])\n",
    "emoji_list = emoji_list[[i.encode('raw_unicode_escape') for i in DV.feature_names_]]\n",
    "emoji_df=pd.DataFrame(emoji_df.toarray(), columns=[i.encode('raw_unicode_escape') for i in DV.feature_names_])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# combine tweet_df with emoji_df\n",
    "tweet_with_emoji_df = tweet_df.copy()\n",
    "tweet_with_emoji_df[emoji_df.columns] = emoji_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>\\u2639</th>\n",
       "      <th>\\u263a</th>\n",
       "      <th>\\U0001f600</th>\n",
       "      <th>\\U0001f601</th>\n",
       "      <th>\\U0001f602</th>\n",
       "      <th>\\U0001f603</th>\n",
       "      <th>\\U0001f604</th>\n",
       "      <th>\\U0001f606</th>\n",
       "      <th>\\U0001f607</th>\n",
       "      <th>...</th>\n",
       "      <th>\\U0001f626</th>\n",
       "      <th>\\U0001f627</th>\n",
       "      <th>\\U0001f628</th>\n",
       "      <th>\\U0001f629</th>\n",
       "      <th>\\U0001f62d</th>\n",
       "      <th>\\U0001f630</th>\n",
       "      <th>\\U0001f631</th>\n",
       "      <th>\\U0001f638</th>\n",
       "      <th>\\U0001f641</th>\n",
       "      <th>\\U0001f642</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bro that's so depressing.. Let's go to his house 🙄\\n</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>thank you thank you for telling me about them omg i love them ❤x\\n</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Excellent day at work, laughed all day. Blessed to do my job. #crewlife #averybritishairline #post2276gettingthere 👏🏻 \\n</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Things are getting there \\n</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I'm so proud of you! Welcome to the team!! #BellLetsTalk #SickNotWeak   \\n</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                      tweet  \\\n",
       "0                                                                      bro that's so depressing.. Let's go to his house 🙄\\n   \n",
       "1                                                        thank you thank you for telling me about them omg i love them ❤x\\n   \n",
       "2  Excellent day at work, laughed all day. Blessed to do my job. #crewlife #averybritishairline #post2276gettingthere 👏🏻 \\n   \n",
       "3                                                                                               Things are getting there \\n   \n",
       "4                                                I'm so proud of you! Welcome to the team!! #BellLetsTalk #SickNotWeak   \\n   \n",
       "\n",
       "   \\u2639  \\u263a  \\U0001f600  \\U0001f601  \\U0001f602  \\U0001f603  \\U0001f604  \\\n",
       "0       0       0           0           0           1           0           0   \n",
       "1       0       0           0           0           1           0           0   \n",
       "2       0       0           0           0           0           0           0   \n",
       "3       0       0           0           0           0           0           0   \n",
       "4       0       0           0           0           0           0           0   \n",
       "\n",
       "   \\U0001f606  \\U0001f607     ...      \\U0001f626  \\U0001f627  \\U0001f628  \\\n",
       "0           0           0     ...               0           0           0   \n",
       "1           0           0     ...               0           0           0   \n",
       "2           0           0     ...               0           0           0   \n",
       "3           0           1     ...               0           0           0   \n",
       "4           0           0     ...               0           0           0   \n",
       "\n",
       "   \\U0001f629  \\U0001f62d  \\U0001f630  \\U0001f631  \\U0001f638  \\U0001f641  \\\n",
       "0           0           0           0           0           0           0   \n",
       "1           0           1           0           0           0           0   \n",
       "2           0           0           0           0           0           0   \n",
       "3           0           0           0           0           0           0   \n",
       "4           0           0           0           0           0           0   \n",
       "\n",
       "   \\U0001f642  \n",
       "0           0  \n",
       "1           0  \n",
       "2           0  \n",
       "3           0  \n",
       "4           0  \n",
       "\n",
       "[5 rows x 35 columns]"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet_with_emoji_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>\\u2639</th>\n",
       "      <th>\\u263a</th>\n",
       "      <th>\\U0001f600</th>\n",
       "      <th>\\U0001f601</th>\n",
       "      <th>\\U0001f602</th>\n",
       "      <th>\\U0001f603</th>\n",
       "      <th>\\U0001f604</th>\n",
       "      <th>\\U0001f606</th>\n",
       "      <th>\\U0001f607</th>\n",
       "      <th>\\U0001f60a</th>\n",
       "      <th>...</th>\n",
       "      <th>\\U0001f626</th>\n",
       "      <th>\\U0001f627</th>\n",
       "      <th>\\U0001f628</th>\n",
       "      <th>\\U0001f629</th>\n",
       "      <th>\\U0001f62d</th>\n",
       "      <th>\\U0001f630</th>\n",
       "      <th>\\U0001f631</th>\n",
       "      <th>\\U0001f638</th>\n",
       "      <th>\\U0001f641</th>\n",
       "      <th>\\U0001f642</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>☹</td>\n",
       "      <td>☺</td>\n",
       "      <td>😀</td>\n",
       "      <td>😁</td>\n",
       "      <td>😂</td>\n",
       "      <td>😃</td>\n",
       "      <td>😄</td>\n",
       "      <td>😆</td>\n",
       "      <td>😇</td>\n",
       "      <td>😊</td>\n",
       "      <td>...</td>\n",
       "      <td>😦</td>\n",
       "      <td>😧</td>\n",
       "      <td>😨</td>\n",
       "      <td>😩</td>\n",
       "      <td>😭</td>\n",
       "      <td>😰</td>\n",
       "      <td>😱</td>\n",
       "      <td>😸</td>\n",
       "      <td>🙁</td>\n",
       "      <td>🙂</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  \\u2639 \\u263a \\U0001f600 \\U0001f601 \\U0001f602 \\U0001f603 \\U0001f604  \\\n",
       "0      ☹      ☺          😀          😁          😂          😃          😄   \n",
       "\n",
       "  \\U0001f606 \\U0001f607 \\U0001f60a    ...     \\U0001f626 \\U0001f627  \\\n",
       "0          😆          😇          😊    ...              😦          😧   \n",
       "\n",
       "  \\U0001f628 \\U0001f629 \\U0001f62d \\U0001f630 \\U0001f631 \\U0001f638  \\\n",
       "0          😨          😩          😭          😰          😱          😸   \n",
       "\n",
       "  \\U0001f641 \\U0001f642  \n",
       "0          🙁          🙂  \n",
       "\n",
       "[1 rows x 34 columns]"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emoji_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}