twitter_scrap_old_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Obtaining GetOldTweets3 from git+https://github.com/Mottl/GetOldTweets3#egg=GetOldTweets3\n",
      "  Cloning https://github.com/Mottl/GetOldTweets3 to ./src/getoldtweets3\n",
      "  Running command git clone -q https://github.com/Mottl/GetOldTweets3 /home/aravind/Desktop/Maha/COVID19/src/getoldtweets3\n",
      "Requirement already satisfied: lxml>=3.5.0 in /home/aravind/anaconda3/lib/python3.7/site-packages (from GetOldTweets3) (4.5.0)\n",
      "Collecting pyquery>=1.2.10\n",
      "  Downloading pyquery-1.4.1-py2.py3-none-any.whl (22 kB)\n",
      "Collecting cssselect>0.7.9\n",
      "  Downloading cssselect-1.1.0-py2.py3-none-any.whl (16 kB)\n",
      "Installing collected packages: cssselect, pyquery, GetOldTweets3\n",
      "  Running setup.py develop for GetOldTweets3\n",
      "Successfully installed GetOldTweets3 cssselect-1.1.0 pyquery-1.4.1\n"
     ]
    }
   ],
   "source": [
    "!pip install -e git+https://github.com/Mottl/GetOldTweets3#egg=GetOldTweets3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/aravind/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tweepy\n",
    "import pandas as pd\n",
    "import json\n",
    "import csv\n",
    "from textblob import TextBlob\n",
    "import re\n",
    "import nltk\n",
    "from  geopy.geocoders import Nominatim\n",
    "from datetime import datetime\n",
    "import pycountry\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import GetOldTweets3 as got\n",
    "import pickle\n",
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_tweets(str_strt_date,str_end_date,lst_search_string,filename):\n",
    "    search_string=' '.join([str(elem) for elem in lst_search_string])\n",
    "\n",
    "    tweetCriteria = got.manager.TweetCriteria()\\\n",
    "                        .setQuerySearch(search_string)\\\n",
    "                        .setSince(str_strt_date)\\\n",
    "                        .setUntil(str_end_date)\\\n",
    "                        .setLang('en')\n",
    "\n",
    "    result= got.manager.TweetManager.getTweets(tweetCriteria)\n",
    "    pickle.dump( result,open(filename, \"wb\" ) )\n",
    "    print(\"Done extracting tweets\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_duplicate(data_set):\n",
    "    return data_set.drop_duplicates(subset =[\"id\",\"username\"],keep = False) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform_tweets(file, save_file):\n",
    "    results= pickle.load( open(file, \"rb\" ) )\n",
    "    \n",
    "    id_list = [tweet.id for tweet  in results]\n",
    "    data_set = pd.DataFrame(id_list, columns = [\"id\"],index=None)\n",
    "    data_set[\"username\"] = [tweet.username for tweet in results]\n",
    "    data_set[\"text\"] = [tweet.text for tweet in results]\n",
    "    data_set[\"created_at\"] = [tweet.date for tweet in results]\n",
    "    #extracting the date from the created at column and making a date column\n",
    "    data_set['date'] =  pd.to_datetime(data_set['created_at']).dt.date\n",
    "    data_set[\"retweets\"] = [tweet.retweets for tweet in results]\n",
    "    data_set[\"hashtags\"] = [tweet.hashtags for tweet in results]\n",
    "    data_set[\"geo\"] = [tweet.geo for tweet in results]\n",
    "    data_set=remove_duplicate(data_set)\n",
    "    \n",
    "    polarity_scores = []\n",
    "    subjectivity_scores = []\n",
    "    for tweet in results:\n",
    "        tweets = tweet.text\n",
    "        tweets = ' '.join(re.sub(\"(@[A-Za-z0-9]+) | ({*0-9A-Za-z \\t]) |] (\\wt:\\/\\/\\St+)\", \" \", tweets).split())\n",
    "        tweets = ' '.join(re.sub('RT',' ', tweets).split())  \n",
    "        \n",
    "        blob = TextBlob(tweets.strip())\n",
    "        #global polarity \n",
    "        #global subjectivity \n",
    "\n",
    "        polarity = 0\n",
    "        subjectivity = 0\n",
    "\n",
    "        for sent in blob.sentences:\n",
    "            polarity += round(sent.sentiment.polarity,2)\n",
    "            subjectivity += round(sent.sentiment.subjectivity,2)\n",
    "        polarity = polarity/len(blob.sentences)\n",
    "        subjectivity = subjectivity/len(blob.sentences)\n",
    "        \n",
    "        polarity_scores.append(polarity)\n",
    "        subjectivity_scores.append(subjectivity)\n",
    "    \n",
    "    data_set[\"polarity\"] = polarity_scores\n",
    "    data_set[\"subjectivity\"] = subjectivity_scores\n",
    "\n",
    "    if os.path.exists(save_file):\n",
    "        df=pd.read_csv(save_file)\n",
    "        df_2=pd.concat([data_set,df])\n",
    "        df_2=remove_duplicate(df_2)\n",
    "\n",
    "        df_2.to_csv(save_file)\n",
    "\n",
    "    else:\n",
    "        data_set.to_csv(save_file)\n",
    "\n",
    "    print(\"Done tranforming tweets \\nShape:{}\".format(data_set.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done extracting tweets\n"
     ]
    }
   ],
   "source": [
    "extract_tweets(\"2020-03-01\",\"2020-03-05\",['corona OR corona19 OR Dow OR Nasdaq OR S&P','USA'],\"raw_tweets_mar_1_5.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done tranforming tweets \n",
      "Shape:(2721, 10)\n"
     ]
    }
   ],
   "source": [
    "transform_tweets(\"raw_tweets_mar_1_5.pkl\",\"market_mar_1_5_norm.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done extracting tweets\n",
      "Done tranforming tweets \n",
      "Shape:(7285, 10)\n"
     ]
    }
   ],
   "source": [
    "extract_tweets(\"2020-03-23\",\"2020-03-28\",['corona OR corona19 OR DJI OR Nasdaq OR S&P', 'USA'],\"raw_tweets_count_mar_23_28.pkl\")\n",
    "transform_tweets(\"raw_tweets_count_mar_23_28.pkl\",\"tweets_count_mar_23_28.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done tranforming tweets \n",
      "Shape:(7285, 10)\n"
     ]
    }
   ],
   "source": [
    "transform_tweets(\"raw_tweets_count_mar_23_28.pkl\",\"tweets_count_mar_23_28_norm.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done extracting tweets\n"
     ]
    }
   ],
   "source": [
    "extract_tweets(\"2020-03-17\",\"2020-03-21\",['corona OR corona19 OR Dow OR Nasdaq OR S&P','USA'],\"raw_tweets_mar_17_22.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done tranforming tweets \n",
      "Shape:(8730, 10)\n"
     ]
    }
   ],
   "source": [
    "transform_tweets(\"raw_tweets_mar_17_22.pkl\",\"market_mar_17_21.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 0.00    4874\n",
       " 0.50     192\n",
       " 0.20     136\n",
       " 0.10     117\n",
       "-0.12     103\n",
       "         ... \n",
       "-0.95       1\n",
       " 0.67       1\n",
       " 0.84       1\n",
       "-1.39       1\n",
       " 0.68       1\n",
       "Name: polarity, Length: 392, dtype: int64"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recovery = pd.read_csv('market_mar_17_21.csv')\n",
    "recovery['polarity'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.00    4563\n",
       "1.00     295\n",
       "0.50     271\n",
       "0.40     152\n",
       "0.60     127\n",
       "        ... \n",
       "1.91       1\n",
       "2.57       1\n",
       "1.78       1\n",
       "4.15       1\n",
       "2.45       1\n",
       "Name: subjectivity, Length: 314, dtype: int64"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recovery['subjectivity'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 0.00    1277\n",
       " 0.50      69\n",
       " 0.10      49\n",
       " 0.20      44\n",
       "-0.20      44\n",
       "         ... \n",
       " 0.05       1\n",
       " 1.20       1\n",
       "-2.18       1\n",
       " 1.02       1\n",
       " 1.06       1\n",
       "Name: polarity, Length: 294, dtype: int64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ill = pd.read_csv('market_mar_1_5.csv')\n",
    "ill['polarity'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.00    1161\n",
       "0.50      98\n",
       "1.00      85\n",
       "0.40      54\n",
       "0.60      44\n",
       "        ... \n",
       "3.05       1\n",
       "0.59       1\n",
       "1.18       1\n",
       "1.57       1\n",
       "1.06       1\n",
       "Name: subjectivity, Length: 274, dtype: int64"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ill['subjectivity'].value_counts()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}