tf-idf_methods_compared.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def transform(data):\n",
    "    # transform the current dataframes (as in gn_gender_all.csv) into \n",
    "    # format suitable for \n",
    "    words = list(set(data['word']))\n",
    "    groups = list(set(data['group']))\n",
    "    counts = list()\n",
    "    for gr in groups:\n",
    "        count = list()\n",
    "        data_group = data.loc[data['group']==gr,]\n",
    "        for word in words:\n",
    "            data_word = data_group.loc[data_group['word']==word,]\n",
    "            if list(data_word.shape)[0]>0:\n",
    "                count.append(list(data_word['counts'])[0])\n",
    "            else:\n",
    "                count.append(0)\n",
    "        counts.append(count)\n",
    "    return words,counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def sumColumn(m, column,exp=1):\n",
    "    # calculate column sum of a matrix; exp is the exponent \n",
    "    # so that the result is (sum(column_data^exp)) and set to 1 by default\n",
    "    total = 0\n",
    "    for row in range(len(m)):\n",
    "        total += m[row][column]**exp\n",
    "    return total\n",
    "\n",
    "def sumRow(m, row,exp=1):\n",
    "    # calculate row sum of a matrix; exp is the exponent \n",
    "    # so that the result is (sum(row_data^exp)) and set to 1 by default\n",
    "    total = 0\n",
    "    for col in range(len(m[0])):\n",
    "        total += m[row][col]**exp\n",
    "    return total\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "\n",
    "\n",
    "def tfidf(counts,method='scikit-learn'):\n",
    "    if method=='scikit-learn':\n",
    "        # The scikit-learn method\n",
    "        transformer = TfidfTransformer()\n",
    "        result = list(transformer.fit_transform(counts).toarray())\n",
    "        for row in range(len(counts)):\n",
    "            result[row] = list(result[row])\n",
    "    else:\n",
    "        result = []\n",
    "        for row in range(len(counts)):\n",
    "            result.append([])\n",
    "        for col in range(len(counts[0])):\n",
    "            colSum = sumColumn(counts,col)\n",
    "            for row in range(len(counts)):\n",
    "                if method=='conditional':\n",
    "                    # conditional probability of the entry being in a certain group \n",
    "                    # given that a certain word is observed\n",
    "                    result[row].append(float(counts[row][col])/colSum)\n",
    "                elif method=='log-conditional':\n",
    "                    # similar to conditional, just instead of dividing the total \n",
    "                    # frequency of each word, divide the log of it so that the \n",
    "                    # result is less extreme \n",
    "                    result[row].append(float(counts[row][col])/np.log(colSum))\n",
    "                else:\n",
    "                    print \"method not found\"\n",
    "                    return result\n",
    "        # Then do l2 normalization\n",
    "        for row in range(len(result)):\n",
    "            norm = sumRow(result,row,2)\n",
    "            for col in range(len(result[0])):\n",
    "                result[row][col] = result[row][col]**2/norm\n",
    "                    \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def wordCloudString2(frequency,words,scale=1000):\n",
    "    # Take the frequency calculated from tf-idf to generate a string for \n",
    "    # word cloud making.\n",
    "    wordCloudStr = \"\" \n",
    "    for i in range(len(words)):\n",
    "        word = words[i]\n",
    "        # scale is a constant to be multiplied to the frequency to get the actual\n",
    "        # number of times a word is repeated in the string, so that this number\n",
    "        # is >1 for all words and proportional to their frequencies.\n",
    "        for j in range(int(frequency[i]*scale)):\n",
    "            wordCloudStr += word\n",
    "            wordCloudStr += \" \"\n",
    "    return wordCloudStr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      0\n",
       "1      0\n",
       "2      0\n",
       "3      0\n",
       "4      0\n",
       "5      0\n",
       "6      0\n",
       "7      0\n",
       "8      0\n",
       "9      0\n",
       "10     0\n",
       "11     0\n",
       "12     0\n",
       "13     0\n",
       "14     0\n",
       "15     0\n",
       "16     0\n",
       "17     0\n",
       "18     0\n",
       "19     0\n",
       "20     0\n",
       "21     0\n",
       "22     0\n",
       "23     0\n",
       "24     0\n",
       "25     0\n",
       "26     0\n",
       "27     0\n",
       "28     0\n",
       "29     0\n",
       "      ..\n",
       "270    5\n",
       "271    5\n",
       "272    5\n",
       "273    5\n",
       "274    5\n",
       "275    5\n",
       "276    5\n",
       "277    5\n",
       "278    5\n",
       "279    5\n",
       "280    5\n",
       "281    5\n",
       "282    5\n",
       "283    5\n",
       "284    5\n",
       "285    5\n",
       "286    5\n",
       "287    5\n",
       "288    5\n",
       "289    5\n",
       "290    5\n",
       "291    5\n",
       "292    5\n",
       "293    5\n",
       "294    5\n",
       "295    5\n",
       "296    5\n",
       "297    5\n",
       "298    5\n",
       "299    5\n",
       "Name: group, dtype: int64"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Data = pd.read_csv(\"../gn_age_group_all.csv\")\n",
    "Data['group']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "words,counts = transform(Data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x1 = tfidf(counts,method='scikit')\n",
    "x2 = tfidf(counts,method='conditional')\n",
    "x3 = tfidf(counts,method='log-conditional')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'houston houston houston houston houston houston houston houston houston houston random random random random facts facts facts facts facts facts facts hate hate hate hate hate hate hate hate hate hate hate member member member member member member member simple simple simple simple simple simple simple good good good good good know know know know know know know know know know know lasik lasik lasik lasik lasik lasik lasik lasik lasik like like like like like like like like like like like like like like like like like like like like like banking banking banking banking banking banking banking banking someone someone someone someone someone someone someone someone someone people people people people people people people people people seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo seo home home home home home home home home girl girl girl girl girl girl depression depression depression depression depression depression new new new new new new new new new tips tips tips tips tips tips tips men men men men men men use use use use use use use vitamin vitamin vitamin vitamin vitamin vitamin vitamin vitamin vitamin vitamin vitamin heating heating heating heating heating heating heating heating heating heating heating experience experience experience experience experience think think think think think think think think love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love love feel feel feel feel feel feel top top top top top top top top live live live live music music music music music music music music music music music music life life life life life life known known known known known known known known believe believe believe believe believe woman woman woman woman woman woman woman woman project project project project project project guide guide guide guide guide guide guide guide guide guide single single single single single single want want want want want want want want want want want want want want want want want want want want want want want want want want want want want need need need need need need need need need need need serum serum serum serum serum serum serum serum serum serum serum things things things things things get get get get get write write write write write best best best best best best best best best offshore offshore offshore offshore offshore offshore offshore offshore offshore offshore offshore offshore offshore offshore offshore quote quote quote quote friends friends friends friends thoughts thoughts thoughts thoughts person person person person person person person person person talk talk talk talk talk talk talk '"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordCloudString2(x1[1],words,100)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}