Feature_Engineering_Sentiment_Analysis_Scotref2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Feature_Engineering - Sentiment_Analysis_Scotref2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gracecarrillo/Political-Data-Science/blob/master/Feature_Engineering_Sentiment_Analysis_Scotref2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ma5cRQJafwRY",
        "colab_type": "text"
      },
      "source": [
        "# Scottish independence: Twitter data Sentiment Analysis\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mFiPGpFfkIUU",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "## 4. Feature Engineering\n",
        "  \n",
        "  - Sentiment Score with Vader\n",
        "  - Part of Speech Tags (POS)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oP1c5gDe2M6-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Must be upgraded\n",
        "!pip install tqdm==4.36.1 --upgrade"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v0Xc0nZtWke9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install --upgrade gensim"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VOnd_sShyvQx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install vaderSentiment"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rk4fAdKjRIdC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cpXSoADwxTTs",
        "colab_type": "code",
        "outputId": "a6a2e844-2fa2-4821-aaa6-3666babd0a70",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        }
      },
      "source": [
        "# general\n",
        "import os\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import csv\n",
        "import string\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import random\n",
        "import itertools\n",
        "import collections\n",
        "from collections import Counter\n",
        "\n",
        "# tweets\n",
        "import tweepy as tw\n",
        "import re\n",
        "from collections import Counter\n",
        "from string import punctuation\n",
        "from tweepy import OAuthHandler\n",
        "import json\n",
        "\n",
        "# text manipulation \n",
        "import nltk \n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import TweetTokenizer\n",
        "from nltk.stem.wordnet import WordNetLemmatizer\n",
        "from nltk.stem.porter import PorterStemmer\n",
        "from nltk.stem.porter import * \n",
        "\n",
        "# plots\n",
        "from wordcloud import WordCloud\n",
        "import plotly\n",
        "import chart_studio.plotly as py\n",
        "import plotly.graph_objs as go \n",
        "from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\n",
        "import cufflinks as cf\n",
        "cf.go_offline()\n",
        "\n",
        "# Feature Engineering\n",
        "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
        "\n",
        "# Machine Learning\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.pipeline import FeatureUnion\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
        "from sklearn.model_selection import KFold, cross_val_score\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.svm import LinearSVC\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.externals import joblib\n",
        "from sklearn.base import BaseEstimator, TransformerMixin\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer, HashingVectorizer, CountVectorizer\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "\n",
        "# For geoplots\n",
        "from IPython.display import IFrame\n",
        "import folium\n",
        "from folium import plugins\n",
        "from folium.plugins import MarkerCluster, FastMarkerCluster, HeatMapWithTime\n",
        "import networkx\n",
        "\n",
        "# hide warnings\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "\n",
        "# set plot preferences\n",
        "plt.style.use(style='ggplot')\n",
        "plt.rcParams['figure.figsize'] = (10, 6)\n",
        "pd.set_option(\"display.max_colwidth\", 200) \n",
        "\n",
        "print('Libraries imported')\n",
        "%matplotlib inline"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "        <script type=\"text/javascript\">\n",
              "        window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
              "        if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
              "        if (typeof require !== 'undefined') {\n",
              "        require.undef(\"plotly\");\n",
              "        requirejs.config({\n",
              "            paths: {\n",
              "                'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n",
              "            }\n",
              "        });\n",
              "        require(['plotly'], function(Plotly) {\n",
              "            window._Plotly = Plotly;\n",
              "        });\n",
              "        }\n",
              "        </script>\n",
              "        "
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/externals/joblib/__init__.py:15: FutureWarning:\n",
            "\n",
            "sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Libraries imported\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k6F-pteWPrf5",
        "colab_type": "text"
      },
      "source": [
        "## 4. Feature Engineering\n",
        "\n",
        "To analyse a preprocessed data, it needs to be converted into features. Depending upon the usage, text features can be constructed using assorted techniques like Bag of Words, TF-IDF, and Word Embeddings. \n",
        "\n",
        "A basic approach to Bag of Words will not be able to capture the difference between “I like you”, where “like” is a verb with a positive sentiment, and “I am like you”, where “like” is a preposition with a neutral sentiment.\n",
        "\n",
        "To improve this technique we'll extract features using Vader's Polarity Scores and Part of Speech (POS) tags.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F-WC01DIyaMB",
        "colab_type": "text"
      },
      "source": [
        "### 4.1 Sentiment Score with Vader\n",
        "\n",
        "Vader sentiment analysis tool belongs to a type of sentiment analysis that is based on lexicons of sentiment-related words. It uses a bag of words approach (a lookup table of positive and negative words but in this approach, each of the words in the lexicon is rated as to whether it is positive or negative, and in many cases, how positive or negative. \n",
        "\n",
        "VADER produces four sentiment metrics. The first three, positive, neutral and negative which is self explanatory. The final metric, the compound score, is the sum of all of the lexicon ratings, which are then standardised to range between -1 and 1. \n",
        "\n",
        "For the compound score:\n",
        "- positive sentiment : (compound score >= 0.05)\n",
        "- neutral sentiment : (compound score > -0.05) and (compound score < 0.05)\n",
        "- negative sentiment : (compound score <= -0.05)\n",
        "\n",
        "We'll use these scores to create features based on the sentiment metrics of our tweets, which will then be used as adittional features for modeling."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oaCGDgmFlgs1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Load data set\n",
        "train = pd.read_csv('/content/drive/My Drive/Twitter_Project/cleaned_train_data.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h0uWjH6dnkHR",
        "colab_type": "code",
        "outputId": "4ee1a3d9-dbfb-4d24-da5d-33b30f61ca1c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 191
        }
      },
      "source": [
        "train.info()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 50000 entries, 0 to 49999\n",
            "Data columns (total 5 columns):\n",
            "label         50000 non-null int64\n",
            "text          50000 non-null object\n",
            "word count    50000 non-null int64\n",
            "tidy_tweet    49685 non-null object\n",
            "tokens        50000 non-null object\n",
            "dtypes: int64(2), object(3)\n",
            "memory usage: 1.9+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7AIREcaioi48",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train.dropna(subset=['tidy_tweet'], inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MRzTReNbQ4mB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "analyser = SentimentIntensityAnalyzer()\n",
        "\n",
        "def polarity_scores_all(tweet):\n",
        "  '''\n",
        "  Takes string of text to:\n",
        "  1. Gets sentiment metrics\n",
        "  2. Returns negative, neutral, positive \n",
        "  and compound scores as lists.\n",
        "  '''\n",
        "  neg, neu, pos, compound = [], [], [], []\n",
        "  analyser = SentimentIntensityAnalyzer()\n",
        "  \n",
        "  for text in tweet:\n",
        "    dict_ = analyser.polarity_scores(text)\n",
        "    neg.append(dict_['neg'])\n",
        "    neu.append(dict_['neu'])\n",
        "    pos.append(dict_['pos'])\n",
        "    compound.append(dict_['compound'])\n",
        "  \n",
        "  return neg, neu, pos, compound"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "avFxGRRW2em_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "all_scores = polarity_scores_all(train.tidy_tweet.values)\n",
        "train['neg_scores'] = all_scores[0]\n",
        "train['neu_scores'] = all_scores[1]\n",
        "train['pos_scores'] = all_scores[2]\n",
        "train['compound_scores'] = all_scores[3]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EK--046ZLhER",
        "colab_type": "code",
        "outputId": "00eb73e1-5215-459a-8f8e-4fe2c9dd94ce",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 275
        }
      },
      "source": [
        "train.head(4)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "      <th>word count</th>\n",
              "      <th>tidy_tweet</th>\n",
              "      <th>tokens</th>\n",
              "      <th>neg_scores</th>\n",
              "      <th>neu_scores</th>\n",
              "      <th>pos_scores</th>\n",
              "      <th>compound_scores</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>4</td>\n",
              "      <td>is it LOVE or BREAD???</td>\n",
              "      <td>5</td>\n",
              "      <td>love bread</td>\n",
              "      <td>['love', 'bread']</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.192</td>\n",
              "      <td>0.808</td>\n",
              "      <td>0.6369</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>now doing the weights again...urgh  my poor laptop is burning up in this summer evening's heat</td>\n",
              "      <td>16</td>\n",
              "      <td>weight urgh poor laptop burn summer even heat</td>\n",
              "      <td>['weights', 'urgh', 'poor', 'laptop', 'burning', 'summer', 'evening', 'heat']</td>\n",
              "      <td>0.307</td>\n",
              "      <td>0.693</td>\n",
              "      <td>0.000</td>\n",
              "      <td>-0.4767</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4</td>\n",
              "      <td>just done my weekly weigh in. on target  woohoo</td>\n",
              "      <td>9</td>\n",
              "      <td>done weekli weigh target woohoo</td>\n",
              "      <td>['done', 'weekly', 'weigh', 'target', 'woohoo']</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.548</td>\n",
              "      <td>0.452</td>\n",
              "      <td>0.5106</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>@Ambluc, cool thanks, having trouble  back replying to anyone, the little arrow is not working</td>\n",
              "      <td>15</td>\n",
              "      <td>cool thank troubl back repli anyon littl arrow work</td>\n",
              "      <td>['cool', 'thanks', 'trouble', 'back', 'replying', 'anyone', 'little', 'arrow', 'working']</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.593</td>\n",
              "      <td>0.407</td>\n",
              "      <td>0.5859</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   label  ... compound_scores\n",
              "0      4  ...          0.6369\n",
              "1      0  ...         -0.4767\n",
              "2      4  ...          0.5106\n",
              "3      0  ...          0.5859\n",
              "\n",
              "[4 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HtNcewVr6y8N",
        "colab_type": "text"
      },
      "source": [
        "### 4.1 Part of Speech Tags (POS)\n",
        "\n",
        "Part of Speech tagging (POS) is where a part of speech is assigned to each word in a list using context clues. This is useful because the same word with a different part of speech can have two completely different meanings. Is the process of marking up a word in a corpus to a corresponding part of a speech tag, based on its context and definition. This task is not straightforward, as a particular word may have a different part of speech based on the context in which the word is used.\n",
        "\n",
        "We'll use a simple lexical based method that assigns the POS tag to the most frequently occurring word in the training corpus and add the tags as features in our model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FJra43QPHMGV",
        "colab_type": "code",
        "outputId": "65214ba1-e3a4-42c4-f730-53885a6c5a01",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        }
      },
      "source": [
        "nltk.download('averaged_perceptron_tagger')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K9eNbZ4h79wy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# To transform pos tags to readable tags\n",
        "pos_family = {  \n",
        "    'NOUN' : ['NN','NNS','NNP'], # Removed 'NNPS'\n",
        "    'PRON' : ['PRP','PRP$','WP','WP$'],\n",
        "    'VERB' : ['VB','VBD','VBG','VBN','VBP','VBZ'],\n",
        "    'ADJ' :  ['JJ','JJR','JJS'],\n",
        "    'ADV' : ['RB','RBR','RBS','WRB']\n",
        "}\n",
        "\n",
        "def count_pos_tag(tweets):\n",
        "  '''\n",
        "  Takes string of text to:\n",
        "  1. Processes text and attaches POS tags\n",
        "  2. Input the dictionary of POS tags into a Counter.\n",
        "  2. Returns list of POS tags with occurrence number '''\n",
        "  total_count = []\n",
        "  for s in tweets:\n",
        "    partial_count = {}\n",
        "    s = s.split()\n",
        "    count_pos = Counter(dict(nltk.pos_tag(s)).values())\n",
        "    \n",
        "    for item, value in count_pos.items():\n",
        "      partial_count[item] = partial_count.get(item, 0) + 1\n",
        "            \n",
        "    total_count.append(partial_count)\n",
        "\n",
        "  return total_count"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zjOxvD3bE9_M",
        "colab_type": "code",
        "outputId": "fc54315d-3b83-4bf9-ca0e-0a20c8e73826",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        }
      },
      "source": [
        "# Retrieve POS tags with occurrence \n",
        "total_count = count_pos_tag(train.tidy_tweet.values)\n",
        "\n",
        "# As dataframe \n",
        "pos_df = pd.DataFrame(total_count)\n",
        "\n",
        "# Remove unwanted characters\n",
        "pos_df = pos_df.drop(['$', 'IN'], axis = 1) #drop '$' if needed\n",
        "\n",
        "# Inspection\n",
        "pos_df.columns"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['NN', 'JJ', 'RB', 'VBN', 'VBD', 'MD', 'VB', 'CD', 'NNS', 'VBP', 'DT',\n",
              "       'CC', 'RBR', 'JJS', 'FW', 'JJR', 'VBZ', 'PRP', 'RP', 'VBG', 'WDT',\n",
              "       'WRB', 'WP', 'UH', 'PRP$', 'RBS', 'NNP', 'EX', 'PDT', 'WP$', 'POS',\n",
              "       '''', 'SYM', 'TO'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C0i6Z3sDFCgw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Change tags to readable tags\n",
        "\n",
        "pos_df['NOUN'] = pos_df[pos_family['NOUN']].sum(axis=1)\n",
        "pos_df['PRON'] = pos_df[pos_family['PRON']].sum(axis=1)\n",
        "pos_df['VERB'] = pos_df[pos_family['VERB']].sum(axis=1)\n",
        "pos_df['ADJ'] = pos_df[pos_family['ADJ']].sum(axis=1)\n",
        "pos_df['ADV'] = pos_df[pos_family['ADV']].sum(axis=1)\n",
        "\n",
        "pos_df = pos_df[['NOUN', 'PRON', 'VERB', 'ADJ', 'ADV']]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JVyWNfKlFFIk",
        "colab_type": "code",
        "outputId": "98b4a645-e2ff-462c-9200-3246574234dc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# Add to end of original data set as new features \n",
        "train = pd.concat([train, pos_df], axis = 1)\n",
        "\n",
        "# Deal with NaN\n",
        "train = train.fillna(value=0.0)\n",
        "\n",
        "#train = train.fillna(value=0.0)\n",
        "train.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(49998, 14)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uRjJUCF_OVd-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Remove duplicates \n",
        "train.drop_duplicates(subset=['tidy_tweet'], inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y8odkpIgULsK",
        "colab_type": "code",
        "outputId": "f1182443-1d8a-4426-a134-d091e0b22619",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 347
        }
      },
      "source": [
        "# Check new features\n",
        "train.info()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 47824 entries, 0 to 49999\n",
            "Data columns (total 14 columns):\n",
            "label              47824 non-null float64\n",
            "text               47824 non-null object\n",
            "word count         47824 non-null float64\n",
            "tidy_tweet         47824 non-null object\n",
            "tokens             47824 non-null object\n",
            "neg_scores         47824 non-null float64\n",
            "neu_scores         47824 non-null float64\n",
            "pos_scores         47824 non-null float64\n",
            "compound_scores    47824 non-null float64\n",
            "NOUN               47824 non-null float64\n",
            "PRON               47824 non-null float64\n",
            "VERB               47824 non-null float64\n",
            "ADJ                47824 non-null float64\n",
            "ADV                47824 non-null float64\n",
            "dtypes: float64(11), object(3)\n",
            "memory usage: 5.5+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3gPt59PEN1eg",
        "colab_type": "code",
        "outputId": "1ac4a651-1e53-4ce0-98b6-a182c16882e8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 445
        }
      },
      "source": [
        "train.head(5)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "      <th>word count</th>\n",
              "      <th>tidy_tweet</th>\n",
              "      <th>tokens</th>\n",
              "      <th>neg_scores</th>\n",
              "      <th>neu_scores</th>\n",
              "      <th>pos_scores</th>\n",
              "      <th>compound_scores</th>\n",
              "      <th>NOUN</th>\n",
              "      <th>PRON</th>\n",
              "      <th>VERB</th>\n",
              "      <th>ADJ</th>\n",
              "      <th>ADV</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>4.0</td>\n",
              "      <td>is it LOVE or BREAD???</td>\n",
              "      <td>5.0</td>\n",
              "      <td>love bread</td>\n",
              "      <td>['love', 'bread']</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.192</td>\n",
              "      <td>0.808</td>\n",
              "      <td>0.6369</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>now doing the weights again...urgh  my poor laptop is burning up in this summer evening's heat</td>\n",
              "      <td>16.0</td>\n",
              "      <td>weight urgh poor laptop burn summer even heat</td>\n",
              "      <td>['weights', 'urgh', 'poor', 'laptop', 'burning', 'summer', 'evening', 'heat']</td>\n",
              "      <td>0.307</td>\n",
              "      <td>0.693</td>\n",
              "      <td>0.000</td>\n",
              "      <td>-0.4767</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4.0</td>\n",
              "      <td>just done my weekly weigh in. on target  woohoo</td>\n",
              "      <td>9.0</td>\n",
              "      <td>done weekli weigh target woohoo</td>\n",
              "      <td>['done', 'weekly', 'weigh', 'target', 'woohoo']</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.548</td>\n",
              "      <td>0.452</td>\n",
              "      <td>0.5106</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.0</td>\n",
              "      <td>@Ambluc, cool thanks, having trouble  back replying to anyone, the little arrow is not working</td>\n",
              "      <td>15.0</td>\n",
              "      <td>cool thank troubl back repli anyon littl arrow work</td>\n",
              "      <td>['cool', 'thanks', 'trouble', 'back', 'replying', 'anyone', 'little', 'arrow', 'working']</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.593</td>\n",
              "      <td>0.407</td>\n",
              "      <td>0.5859</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4.0</td>\n",
              "      <td>@LeiRock lmaoo, oh yeahh ! well im stoooopid happy</td>\n",
              "      <td>9.0</td>\n",
              "      <td>lmaoo yeahh well stoooopid happi</td>\n",
              "      <td>['lmaoo', 'yeahh', 'well', 'stoooopid', 'happy']</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.656</td>\n",
              "      <td>0.344</td>\n",
              "      <td>0.2732</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   label  ...  ADV\n",
              "0    4.0  ...  0.0\n",
              "1    0.0  ...  1.0\n",
              "2    4.0  ...  0.0\n",
              "3    0.0  ...  1.0\n",
              "4    4.0  ...  1.0\n",
              "\n",
              "[5 rows x 14 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TbQ0QSd9VyC9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Saving preprocessed dataset\n",
        "train.to_csv('/content/drive/My Drive/Twitter_Project/feat_eng_train_data.csv', index=False)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}