1.1. clean & filter english tweets.ipynb

{"cells":[{"cell_type":"code","metadata":{"cell_id":"00000-98643fa2-5093-478f-9cbc-085aabaef3c3","output_cleared":false,"source_hash":"7a71bf56","execution_millis":1829,"execution_start":1602090129142},"source":"#importing necessery libraries for future analysis of the dataset\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n%matplotlib inline\nimport seaborn as sns\nnp.random.seed(2020)\nimport nltk\nnltk.download('punkt') # one time execution\nnltk.download('stopwords')\nfrom nltk.corpus import stopwords\nstop_words = set(stopwords.words('english'))\nimport re\nfrom nltk.tokenize import word_tokenize","execution_count":null,"outputs":[{"name":"stderr","text":"[nltk_data] Downloading package punkt to /home/jovyan/nltk_data...\n[nltk_data]   Package punkt is already up-to-date!\n[nltk_data] Downloading package stopwords to /home/jovyan/nltk_data...\n[nltk_data]   Package stopwords is already up-to-date!\n","output_type":"stream"}]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00001-bc1b2cde-4811-4fd4-a89a-9a0b0fb60cfa","output_cleared":false,"source_hash":"42aa37c1","execution_millis":1958,"execution_start":1602090130979},"source":"data = pd.read_csv(\"covid19_tweets.csv\")\ndata.head(3)","execution_count":null,"outputs":[{"output_type":"execute_result","execution_count":2,"data":{"application/vnd.deepnote.dataframe.v2+json":{"row_count":3,"column_count":13,"columns":[{"name":"user_name","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"ᏉᎥ☻լꂅϮ","count":1},{"name":"Tom Basile 🇺🇸","count":1},{"name":"Time4fisticuffs","count":1}]}},{"name":"user_location","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"astroworld","count":1},{"name":"New York, NY","count":1},{"name":"Pewee Valley, KY","count":1}]}},{"name":"user_description","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"wednesday addams as a disney princess keepin it [̲̅$̲̅(̲̅ιοο̲̅)̲̅$̲̅] 👸🏻💚♊️🧀🌵🌃🌙","count":1},{"name":"Husband, Father, Columnist & Commentator. Author of Tough Sell: Fighting the Media War in Iraq. Bush Admin Alum. Newsmax Contributor. Fmr Exec Dir NYSGOP","count":1},{"name":"#Christian #Catholic #Conservative #Reagan #Republican #Capitalist; Sports lover - #BBN #Cincinnati #Reds #Bengals #Trump2020","count":1}]}},{"name":"user_created","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"2017-05-26 05:46:42","count":1},{"name":"2009-04-16 20:06:23","count":1},{"name":"2009-02-28 18:57:41","count":1}]}},{"name":"user_followers","dtype":"int64","stats":{"unique_count":3,"nan_count":0,"min":624,"max":9275,"histogram":[{"bin_start":624,"bin_end":1489.1,"count":1},{"bin_start":1489.1,"bin_end":2354.2,"count":1},{"bin_start":2354.2,"bin_end":3219.3,"count":0},{"bin_start":3219.3,"bin_end":4084.4,"count":0},{"bin_start":4084.4,"bin_end":4949.5,"count":0},{"bin_start":4949.5,"bin_end":5814.6,"count":0},{"bin_start":5814.6,"bin_end":6679.7,"count":0},{"bin_start":6679.7,"bin_end":7544.8,"count":0},{"bin_start":7544.8,"bin_end":8409.900000000001,"count":0},{"bin_start":8409.900000000001,"bin_end":9275,"count":1}]}},{"name":"user_friends","dtype":"int64","stats":{"unique_count":3,"nan_count":0,"min":950,"max":9525,"histogram":[{"bin_start":950,"bin_end":1807.5,"count":2},{"bin_start":1807.5,"bin_end":2665,"count":0},{"bin_start":2665,"bin_end":3522.5,"count":0},{"bin_start":3522.5,"bin_end":4380,"count":0},{"bin_start":4380,"bin_end":5237.5,"count":0},{"bin_start":5237.5,"bin_end":6095,"count":0},{"bin_start":6095,"bin_end":6952.5,"count":0},{"bin_start":6952.5,"bin_end":7810,"count":0},{"bin_start":7810,"bin_end":8667.5,"count":0},{"bin_start":8667.5,"bin_end":9525,"count":1}]}},{"name":"user_favourites","dtype":"int64","stats":{"unique_count":3,"nan_count":0,"min":24,"max":18775,"histogram":[{"bin_start":24,"bin_end":1899.1,"count":1},{"bin_start":1899.1,"bin_end":3774.2,"count":0},{"bin_start":3774.2,"bin_end":5649.299999999999,"count":0},{"bin_start":5649.299999999999,"bin_end":7524.4,"count":1},{"bin_start":7524.4,"bin_end":9399.5,"count":0},{"bin_start":9399.5,"bin_end":11274.599999999999,"count":0},{"bin_start":11274.599999999999,"bin_end":13149.699999999999,"count":0},{"bin_start":13149.699999999999,"bin_end":15024.8,"count":0},{"bin_start":15024.8,"bin_end":16899.899999999998,"count":0},{"bin_start":16899.899999999998,"bin_end":18775,"count":1}]}},{"name":"user_verified","dtype":"bool","stats":{"unique_count":2,"nan_count":0,"categories":[{"name":"False","count":2},{"name":"True","count":1}]}},{"name":"date","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"2020-07-25 12:27:21","count":1},{"name":"2020-07-25 12:27:17","count":1},{"name":"2020-07-25 12:27:14","count":1}]}},{"name":"text","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"If I smelled the scent of hand sanitizers today on someone in the past, I would think they were so intoxicated that… https://t.co/QZvYbrOgb0","count":1},{"name":"Hey @Yankees @YankeesPR and @MLB - wouldn't it have made more sense to have the players pay their respects to the A… https://t.co/1QvW0zgyPu","count":1},{"name":"@diane3443 @wdunlap @realDonaldTrump Trump never once claimed #COVID19 was a hoax. We all claim that this effort to… https://t.co/Jkk8vHWHb3","count":1}]}},{"name":"hashtags","dtype":"object","stats":{"unique_count":1,"nan_count":2,"categories":[{"name":"['COVID19']","count":1},{"name":"Missing","count":2}]}},{"name":"source","dtype":"object","stats":{"unique_count":2,"nan_count":0,"categories":[{"name":"Twitter for Android","count":2},{"name":"Twitter for iPhone","count":1}]}},{"name":"is_retweet","dtype":"bool","stats":{"unique_count":1,"nan_count":0,"categories":[{"name":"False","count":3}]}},{"name":"_deepnote_index_column","dtype":"int64"}],"rows_top":[{"user_name":"ᏉᎥ☻լꂅϮ","user_location":"astroworld","user_description":"wednesday addams as a disney princess keepin it [̲̅$̲̅(̲̅ιοο̲̅)̲̅$̲̅] 👸🏻💚♊️🧀🌵🌃🌙","user_created":"2017-05-26 05:46:42","user_followers":624,"user_friends":950,"user_favourites":18775,"user_verified":false,"date":"2020-07-25 12:27:21","text":"If I smelled the scent of hand sanitizers today on someone in the past, I would think they were so intoxicated that… https://t.co/QZvYbrOgb0","hashtags":"nan","source":"Twitter for iPhone","is_retweet":false,"_deepnote_index_column":0},{"user_name":"Tom Basile 🇺🇸","user_location":"New York, NY","user_description":"Husband, Father, Columnist & Commentator. Author of Tough Sell: Fighting the Media War in Iraq. Bush Admin Alum. Newsmax Contributor. Fmr Exec Dir NYSGOP","user_created":"2009-04-16 20:06:23","user_followers":2253,"user_friends":1677,"user_favourites":24,"user_verified":true,"date":"2020-07-25 12:27:17","text":"Hey @Yankees @YankeesPR and @MLB - wouldn't it have made more sense to have the players pay their respects to the A… https://t.co/1QvW0zgyPu","hashtags":"nan","source":"Twitter for Android","is_retweet":false,"_deepnote_index_column":1},{"user_name":"Time4fisticuffs","user_location":"Pewee Valley, KY","user_description":"#Christian #Catholic #Conservative #Reagan #Republican #Capitalist; Sports lover - #BBN #Cincinnati #Reds #Bengals #Trump2020","user_created":"2009-02-28 18:57:41","user_followers":9275,"user_friends":9525,"user_favourites":7254,"user_verified":false,"date":"2020-07-25 12:27:14","text":"@diane3443 @wdunlap @realDonaldTrump Trump never once claimed #COVID19 was a hoax. We all claim that this effort to… https://t.co/Jkk8vHWHb3","hashtags":"['COVID19']","source":"Twitter for Android","is_retweet":false,"_deepnote_index_column":2}],"rows_bottom":null},"text/plain":"         user_name     user_location  \\\n0           ᏉᎥ☻լꂅϮ        astroworld   \n1    Tom Basile 🇺🇸      New York, NY   \n2  Time4fisticuffs  Pewee Valley, KY   \n\n                                    user_description         user_created  \\\n0  wednesday addams as a disney princess keepin i...  2017-05-26 05:46:42   \n1  Husband, Father, Columnist & Commentator. Auth...  2009-04-16 20:06:23   \n2  #Christian #Catholic #Conservative #Reagan #Re...  2009-02-28 18:57:41   \n\n   user_followers  user_friends  user_favourites  user_verified  \\\n0             624           950            18775          False   \n1            2253          1677               24           True   \n2            9275          9525             7254          False   \n\n                  date                                               text  \\\n0  2020-07-25 12:27:21  If I smelled the scent of hand sanitizers toda...   \n1  2020-07-25 12:27:17  Hey @Yankees @YankeesPR and @MLB - wouldn't it...   \n2  2020-07-25 12:27:14  @diane3443 @wdunlap @realDonaldTrump Trump nev...   \n\n      hashtags               source  is_retweet  \n0          NaN   Twitter for iPhone       False  \n1          NaN  Twitter for Android       False  \n2  ['COVID19']  Twitter for Android       False  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_name</th>\n      <th>user_location</th>\n      <th>user_description</th>\n      <th>user_created</th>\n      <th>user_followers</th>\n      <th>user_friends</th>\n      <th>user_favourites</th>\n      <th>user_verified</th>\n      <th>date</th>\n      <th>text</th>\n      <th>hashtags</th>\n      <th>source</th>\n      <th>is_retweet</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>ᏉᎥ☻լꂅϮ</td>\n      <td>astroworld</td>\n      <td>wednesday addams as a disney princess keepin i...</td>\n      <td>2017-05-26 05:46:42</td>\n      <td>624</td>\n      <td>950</td>\n      <td>18775</td>\n      <td>False</td>\n      <td>2020-07-25 12:27:21</td>\n      <td>If I smelled the scent of hand sanitizers toda...</td>\n      <td>NaN</td>\n      <td>Twitter for iPhone</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Tom Basile 🇺🇸</td>\n      <td>New York, NY</td>\n      <td>Husband, Father, Columnist &amp; Commentator. Auth...</td>\n      <td>2009-04-16 20:06:23</td>\n      <td>2253</td>\n      <td>1677</td>\n      <td>24</td>\n      <td>True</td>\n      <td>2020-07-25 12:27:17</td>\n      <td>Hey @Yankees @YankeesPR and @MLB - wouldn't it...</td>\n      <td>NaN</td>\n      <td>Twitter for Android</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Time4fisticuffs</td>\n      <td>Pewee Valley, KY</td>\n      <td>#Christian #Catholic #Conservative #Reagan #Re...</td>\n      <td>2009-02-28 18:57:41</td>\n      <td>9275</td>\n      <td>9525</td>\n      <td>7254</td>\n      <td>False</td>\n      <td>2020-07-25 12:27:14</td>\n      <td>@diane3443 @wdunlap @realDonaldTrump Trump nev...</td>\n      <td>['COVID19']</td>\n      <td>Twitter for Android</td>\n      <td>False</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00002-e807fd8c-002f-46ab-b8dd-63ac35b7f16d","output_cleared":false,"source_hash":"2007f58","execution_millis":3,"execution_start":1602090132947},"source":"data.shape","execution_count":null,"outputs":[{"output_type":"execute_result","execution_count":3,"data":{"text/plain":"(179108, 13)"},"metadata":{}}]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00003-1f002fbb-545f-4db2-b7e3-584a43eaca64","output_cleared":false,"source_hash":"ae3db336","execution_millis":13,"execution_start":1602090132955},"source":"data.dtypes","execution_count":null,"outputs":[{"output_type":"execute_result","execution_count":4,"data":{"text/plain":"user_name           object\nuser_location       object\nuser_description    object\nuser_created        object\nuser_followers       int64\nuser_friends         int64\nuser_favourites      int64\nuser_verified         bool\ndate                object\ntext                object\nhashtags            object\nsource              object\nis_retweet            bool\ndtype: object"},"metadata":{}}]},{"cell_type":"markdown","source":"## Data cleaning","metadata":{"tags":[],"cell_id":"00004-a0cedbc7-b622-4c26-b106-89d2b004baf1","output_cleared":false}},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00005-675b84ce-2e3c-4b20-947b-f46cf7ee2dbe","output_cleared":false,"source_hash":"977202f3","execution_millis":1256,"execution_start":1602090132974},"source":"def get_mention(txt):\n    mention = []\n    for i in txt.split(\" \"):\n        if len(i) > 0 and i[0] == \"@\":\n            mention.append(i)\n    return \"\".join([mention[i] + \", \" if i != len(mention) - 1 else mention[i] for i in range(len(mention))])\n\ndata[\"mention\"] = data.text.apply(get_mention)","execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00005-a162eaa9-d664-4ced-a5cb-ec7d0d54c3a9","output_cleared":false,"source_hash":"a81a93c8","execution_millis":2120,"execution_start":1602090134265},"source":"def remove_link_email(txt):\n    txt = txt.replace(\"...\", \"\")\n    txt = re.sub(r\"http\\S+\", \"\", txt)\n    txt = txt.replace('\\S*@\\S*\\s?', \"\")\n    txt = re.sub(r'[^\\w\\s]', '', txt)\n    return txt\n\ndata.text = data.text.apply(remove_link_email)","execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00005-a81bcf34-003a-4832-9631-e0b93dc95984","output_cleared":false,"source_hash":"85f89041","execution_millis":47466,"execution_start":1602090136400},"source":"# no non-english\ndef clean_non_english(txt): \n    try: \n        txt = re.sub(r'\\W+', ' ', txt)\n        txt = txt.lower()\n        txt = txt.replace(\"[^a-zA-Z]\", \" \")\n        word_tokens = word_tokenize(txt) \n        filtered_word = [w for w in word_tokens if all(ord(c) < 128 for c in w)]\n        filtered_word = [w + \" \" for w in filtered_word]\n        return \"\".join(filtered_word)\n    except:\n        return np.nan\n\ndata[\"english_text\"] = data.text.apply(clean_non_english)","execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00006-41dc23c9-9e0f-4f7b-bf38-677541437308","output_cleared":false,"source_hash":"bc52bcbe","execution_millis":39938,"execution_start":1602090183879},"source":"# all clean\ndef clean_text(english_txt): \n    try: \n        word_tokens = word_tokenize(english_txt)\n        filtered_word = [w for w in word_tokens if not w in stop_words] \n        filtered_word = [w + \" \" for w in filtered_word]\n        return \"\".join(filtered_word)\n    except:\n        return np.nan\n\ndata[\"cleaned_text\"] = data.english_text.apply(clean_text)","execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00007-a2422d78-087b-44ba-97e1-afb9ec635767","output_cleared":false,"source_hash":"878de0e4","execution_millis":238,"execution_start":1602090223973},"source":"data.isnull().sum()","execution_count":null,"outputs":[{"output_type":"execute_result","execution_count":9,"data":{"text/plain":"user_name               0\nuser_location       36771\nuser_description    10286\nuser_created            0\nuser_followers          0\nuser_friends            0\nuser_favourites         0\nuser_verified           0\ndate                    0\ntext                    0\nhashtags            51334\nsource                 77\nis_retweet              0\nmention                 0\nenglish_text            0\ncleaned_text            0\ndtype: int64"},"metadata":{}}]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00007-cf4b2859-727a-4137-828d-845314693b06","output_cleared":false,"source_hash":"107df4c6","execution_millis":278,"execution_start":1602090224239},"source":"def clean_tag(txt):\n    try:\n        chars = \"'[]\"\n        for char in chars:\n            txt = txt.replace(char, \"\")\n        txt = txt.lower()\n        return txt\n    except: \n        return np.nan\n\ndata[\"cleaned_tags\"] = data.hashtags.apply(clean_tag)","execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00007-f142a3c0-520c-4efb-b36a-aa88fea8d1d6","output_cleared":false,"source_hash":"af5d7ed9","execution_millis":189,"execution_start":1602090224606},"source":"lst = []\nfor item in data.cleaned_tags:\n    try:\n        if item != np.nan:\n            lst_word = item.split(\", \")\n        lst += lst_word\n    except:\n        pass\n\nfrom collections import Counter\nx = Counter(lst)\nx.most_common(10)","execution_count":null,"outputs":[{"output_type":"execute_result","execution_count":11,"data":{"text/plain":"[('covid19', 100312),\n ('coronavirus', 10197),\n ('pandemic', 1625),\n ('covid', 1299),\n ('india', 1193),\n ('corona', 1162),\n ('trump', 1101),\n ('lockdown', 963),\n ('coronaviruspandemic', 882),\n ('covid_19', 828)]"},"metadata":{}}]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00007-1b510875-29c9-4d2a-b308-4ee9be075a5c","output_cleared":false,"source_hash":"6566b86d","execution_millis":183,"execution_start":1602090224803},"source":"def get_len_hashtag(txt):\n    try: \n        return len(txt.split(\",\"))\n    except:\n        return np.nan\n\ndata[\"len_hashtag\"] = data.hashtags.apply(get_len_hashtag)","execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00005-98d5fd40-cacd-402f-8337-8bbad76e3e09","output_cleared":false,"source_hash":"829c5011","execution_millis":312,"execution_start":1602090225197},"source":"data.dropna(subset=[\"user_description\", \"user_location\", \"hashtags\", \"cleaned_text\", \"text\", \"english_text\", 'cleaned_tags'], inplace=True)","execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00010-7bfcfef7-3e2a-45bb-ad5d-8c6bc6159ff0","output_cleared":false,"source_hash":"da627bf6","execution_millis":6,"execution_start":1602090225602},"source":"data.head(3)","execution_count":null,"outputs":[{"output_type":"execute_result","execution_count":14,"data":{"application/vnd.deepnote.dataframe.v2+json":{"row_count":3,"column_count":18,"columns":[{"name":"user_name","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"Time4fisticuffs","count":1},{"name":"ethel mertz","count":1},{"name":"DIPR-J&K","count":1}]}},{"name":"user_location","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"Pewee Valley, KY","count":1},{"name":"Stuck in the Middle ","count":1},{"name":"Jammu and Kashmir","count":1}]}},{"name":"user_description","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"#Christian #Catholic #Conservative #Reagan #Republican #Capitalist; Sports lover - #BBN #Cincinnati #Reds #Bengals #Trump2020","count":1},{"name":"#Browns #Indians #ClevelandProud #[]_[] #Cavs #Resist","count":1},{"name":"🖊️Official Twitter handle of Department of Information and Public Relations, Govt of Jammu & Kashmir","count":1}]}},{"name":"user_created","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"2009-02-28 18:57:41","count":1},{"name":"2019-03-07 01:45:06","count":1},{"name":"2017-02-12 06:45:15","count":1}]}},{"name":"user_followers","dtype":"int64","stats":{"unique_count":3,"nan_count":0,"min":197,"max":101009,"histogram":[{"bin_start":197,"bin_end":10278.2,"count":2},{"bin_start":10278.2,"bin_end":20359.4,"count":0},{"bin_start":20359.4,"bin_end":30440.600000000002,"count":0},{"bin_start":30440.600000000002,"bin_end":40521.8,"count":0},{"bin_start":40521.8,"bin_end":50603,"count":0},{"bin_start":50603,"bin_end":60684.200000000004,"count":0},{"bin_start":60684.200000000004,"bin_end":70765.40000000001,"count":0},{"bin_start":70765.40000000001,"bin_end":80846.6,"count":0},{"bin_start":80846.6,"bin_end":90927.8,"count":0},{"bin_start":90927.8,"bin_end":101009,"count":1}]}},{"name":"user_friends","dtype":"int64","stats":{"unique_count":3,"nan_count":0,"min":168,"max":9525,"histogram":[{"bin_start":168,"bin_end":1103.7,"count":2},{"bin_start":1103.7,"bin_end":2039.4,"count":0},{"bin_start":2039.4,"bin_end":2975.1000000000004,"count":0},{"bin_start":2975.1000000000004,"bin_end":3910.8,"count":0},{"bin_start":3910.8,"bin_end":4846.5,"count":0},{"bin_start":4846.5,"bin_end":5782.200000000001,"count":0},{"bin_start":5782.200000000001,"bin_end":6717.900000000001,"count":0},{"bin_start":6717.900000000001,"bin_end":7653.6,"count":0},{"bin_start":7653.6,"bin_end":8589.300000000001,"count":0},{"bin_start":8589.300000000001,"bin_end":9525,"count":1}]}},{"name":"user_favourites","dtype":"int64","stats":{"unique_count":3,"nan_count":0,"min":101,"max":7254,"histogram":[{"bin_start":101,"bin_end":816.3,"count":1},{"bin_start":816.3,"bin_end":1531.6,"count":1},{"bin_start":1531.6,"bin_end":2246.8999999999996,"count":0},{"bin_start":2246.8999999999996,"bin_end":2962.2,"count":0},{"bin_start":2962.2,"bin_end":3677.5,"count":0},{"bin_start":3677.5,"bin_end":4392.799999999999,"count":0},{"bin_start":4392.799999999999,"bin_end":5108.099999999999,"count":0},{"bin_start":5108.099999999999,"bin_end":5823.4,"count":0},{"bin_start":5823.4,"bin_end":6538.7,"count":0},{"bin_start":6538.7,"bin_end":7254,"count":1}]}},{"name":"user_verified","dtype":"bool","stats":{"unique_count":1,"nan_count":0,"categories":[{"name":"False","count":3}]}},{"name":"date","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"2020-07-25 12:27:14","count":1},{"name":"2020-07-25 12:27:10","count":1},{"name":"2020-07-25 12:27:08","count":1}]}},{"name":"text","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"diane3443 wdunlap realDonaldTrump Trump never once claimed COVID19 was a hoax We all claim that this effort to ","count":1},{"name":"brookbanktv The one gift COVID19 has give me is an appreciation for the simple things that were always around me ","count":1},{"name":"25 July  Media Bulletin on Novel CoronaVirusUpdates COVID19 \nkansalrohit69 DrSyedSehrish airnewsalerts ANI ","count":1}]}},{"name":"hashtags","dtype":"object","stats":{"unique_count":2,"nan_count":0,"categories":[{"name":"['COVID19']","count":2},{"name":"['CoronaVirusUpdates', 'COVID19']","count":1}]}},{"name":"source","dtype":"object","stats":{"unique_count":2,"nan_count":0,"categories":[{"name":"Twitter for Android","count":2},{"name":"Twitter for iPhone","count":1}]}},{"name":"is_retweet","dtype":"bool","stats":{"unique_count":1,"nan_count":0,"categories":[{"name":"False","count":3}]}},{"name":"mention","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"@diane3443, @wdunlap, @realDonaldTrump","count":1},{"name":"@brookbanktv","count":1},{"name":"@DrSyedSehrish, @airnewsalerts, @ANI…","count":1}]}},{"name":"english_text","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"diane3443 wdunlap realdonaldtrump trump never once claimed covid19 was a hoax we all claim that this effort to ","count":1},{"name":"brookbanktv the one gift covid19 has give me is an appreciation for the simple things that were always around me ","count":1},{"name":"25 july media bulletin on novel coronavirusupdates covid19 kansalrohit69 drsyedsehrish airnewsalerts ani ","count":1}]}},{"name":"cleaned_text","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"diane3443 wdunlap realdonaldtrump trump never claimed covid19 hoax claim effort ","count":1},{"name":"brookbanktv one gift covid19 give appreciation simple things always around ","count":1},{"name":"25 july media bulletin novel coronavirusupdates covid19 kansalrohit69 drsyedsehrish airnewsalerts ani ","count":1}]}},{"name":"cleaned_tags","dtype":"object","stats":{"unique_count":2,"nan_count":0,"categories":[{"name":"covid19","count":2},{"name":"coronavirusupdates, covid19","count":1}]}},{"name":"len_hashtag","dtype":"float64","stats":{"unique_count":2,"nan_count":0,"min":1,"max":2,"histogram":[{"bin_start":1,"bin_end":1.1,"count":2},{"bin_start":1.1,"bin_end":1.2,"count":0},{"bin_start":1.2,"bin_end":1.3,"count":0},{"bin_start":1.3,"bin_end":1.4,"count":0},{"bin_start":1.4,"bin_end":1.5,"count":0},{"bin_start":1.5,"bin_end":1.6,"count":0},{"bin_start":1.6,"bin_end":1.7000000000000002,"count":0},{"bin_start":1.7000000000000002,"bin_end":1.8,"count":0},{"bin_start":1.8,"bin_end":1.9,"count":0},{"bin_start":1.9,"bin_end":2,"count":1}]}},{"name":"_deepnote_index_column","dtype":"int64"}],"rows_top":[{"user_name":"Time4fisticuffs","user_location":"Pewee Valley, KY","user_description":"#Christian #Catholic #Conservative #Reagan #Republican #Capitalist; Sports lover - #BBN #Cincinnati #Reds #Bengals #Trump2020","user_created":"2009-02-28 18:57:41","user_followers":9275,"user_friends":9525,"user_favourites":7254,"user_verified":false,"date":"2020-07-25 12:27:14","text":"diane3443 wdunlap realDonaldTrump Trump never once claimed COVID19 was a hoax We all claim that this effort to ","hashtags":"['COVID19']","source":"Twitter for Android","is_retweet":false,"mention":"@diane3443, @wdunlap, @realDonaldTrump","english_text":"diane3443 wdunlap realdonaldtrump trump never once claimed covid19 was a hoax we all claim that this effort to ","cleaned_text":"diane3443 wdunlap realdonaldtrump trump never claimed covid19 hoax claim effort ","cleaned_tags":"covid19","len_hashtag":1,"_deepnote_index_column":2},{"user_name":"ethel mertz","user_location":"Stuck in the Middle ","user_description":"#Browns #Indians #ClevelandProud #[]_[] #Cavs #Resist","user_created":"2019-03-07 01:45:06","user_followers":197,"user_friends":987,"user_favourites":1488,"user_verified":false,"date":"2020-07-25 12:27:10","text":"brookbanktv The one gift COVID19 has give me is an appreciation for the simple things that were always around me ","hashtags":"['COVID19']","source":"Twitter for iPhone","is_retweet":false,"mention":"@brookbanktv","english_text":"brookbanktv the one gift covid19 has give me is an appreciation for the simple things that were always around me ","cleaned_text":"brookbanktv one gift covid19 give appreciation simple things always around ","cleaned_tags":"covid19","len_hashtag":1,"_deepnote_index_column":3},{"user_name":"DIPR-J&K","user_location":"Jammu and Kashmir","user_description":"🖊️Official Twitter handle of Department of Information and Public Relations, Govt of Jammu & Kashmir","user_created":"2017-02-12 06:45:15","user_followers":101009,"user_friends":168,"user_favourites":101,"user_verified":false,"date":"2020-07-25 12:27:08","text":"25 July  Media Bulletin on Novel CoronaVirusUpdates COVID19 \nkansalrohit69 DrSyedSehrish airnewsalerts ANI ","hashtags":"['CoronaVirusUpdates', 'COVID19']","source":"Twitter for Android","is_retweet":false,"mention":"@DrSyedSehrish, @airnewsalerts, @ANI…","english_text":"25 july media bulletin on novel coronavirusupdates covid19 kansalrohit69 drsyedsehrish airnewsalerts ani ","cleaned_text":"25 july media bulletin novel coronavirusupdates covid19 kansalrohit69 drsyedsehrish airnewsalerts ani ","cleaned_tags":"coronavirusupdates, covid19","len_hashtag":2,"_deepnote_index_column":4}],"rows_bottom":null},"text/plain":"         user_name         user_location  \\\n2  Time4fisticuffs      Pewee Valley, KY   \n3      ethel mertz  Stuck in the Middle    \n4         DIPR-J&K     Jammu and Kashmir   \n\n                                    user_description         user_created  \\\n2  #Christian #Catholic #Conservative #Reagan #Re...  2009-02-28 18:57:41   \n3  #Browns #Indians #ClevelandProud #[]_[] #Cavs ...  2019-03-07 01:45:06   \n4  🖊️Official Twitter handle of Department of Inf...  2017-02-12 06:45:15   \n\n   user_followers  user_friends  user_favourites  user_verified  \\\n2            9275          9525             7254          False   \n3             197           987             1488          False   \n4          101009           168              101          False   \n\n                  date                                               text  \\\n2  2020-07-25 12:27:14  diane3443 wdunlap realDonaldTrump Trump never ...   \n3  2020-07-25 12:27:10  brookbanktv The one gift COVID19 has give me i...   \n4  2020-07-25 12:27:08  25 July  Media Bulletin on Novel CoronaVirusUp...   \n\n                            hashtags               source  is_retweet  \\\n2                        ['COVID19']  Twitter for Android       False   \n3                        ['COVID19']   Twitter for iPhone       False   \n4  ['CoronaVirusUpdates', 'COVID19']  Twitter for Android       False   \n\n                                  mention  \\\n2  @diane3443, @wdunlap, @realDonaldTrump   \n3                            @brookbanktv   \n4   @DrSyedSehrish, @airnewsalerts, @ANI…   \n\n                                        english_text  \\\n2  diane3443 wdunlap realdonaldtrump trump never ...   \n3  brookbanktv the one gift covid19 has give me i...   \n4  25 july media bulletin on novel coronavirusupd...   \n\n                                        cleaned_text  \\\n2  diane3443 wdunlap realdonaldtrump trump never ...   \n3  brookbanktv one gift covid19 give appreciation...   \n4  25 july media bulletin novel coronavirusupdate...   \n\n                  cleaned_tags  len_hashtag  \n2                      covid19          1.0  \n3                      covid19          1.0  \n4  coronavirusupdates, covid19          2.0  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_name</th>\n      <th>user_location</th>\n      <th>user_description</th>\n      <th>user_created</th>\n      <th>user_followers</th>\n      <th>user_friends</th>\n      <th>user_favourites</th>\n      <th>user_verified</th>\n      <th>date</th>\n      <th>text</th>\n      <th>hashtags</th>\n      <th>source</th>\n      <th>is_retweet</th>\n      <th>mention</th>\n      <th>english_text</th>\n      <th>cleaned_text</th>\n      <th>cleaned_tags</th>\n      <th>len_hashtag</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>2</th>\n      <td>Time4fisticuffs</td>\n      <td>Pewee Valley, KY</td>\n      <td>#Christian #Catholic #Conservative #Reagan #Re...</td>\n      <td>2009-02-28 18:57:41</td>\n      <td>9275</td>\n      <td>9525</td>\n      <td>7254</td>\n      <td>False</td>\n      <td>2020-07-25 12:27:14</td>\n      <td>diane3443 wdunlap realDonaldTrump Trump never ...</td>\n      <td>['COVID19']</td>\n      <td>Twitter for Android</td>\n      <td>False</td>\n      <td>@diane3443, @wdunlap, @realDonaldTrump</td>\n      <td>diane3443 wdunlap realdonaldtrump trump never ...</td>\n      <td>diane3443 wdunlap realdonaldtrump trump never ...</td>\n      <td>covid19</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>ethel mertz</td>\n      <td>Stuck in the Middle</td>\n      <td>#Browns #Indians #ClevelandProud #[]_[] #Cavs ...</td>\n      <td>2019-03-07 01:45:06</td>\n      <td>197</td>\n      <td>987</td>\n      <td>1488</td>\n      <td>False</td>\n      <td>2020-07-25 12:27:10</td>\n      <td>brookbanktv The one gift COVID19 has give me i...</td>\n      <td>['COVID19']</td>\n      <td>Twitter for iPhone</td>\n      <td>False</td>\n      <td>@brookbanktv</td>\n      <td>brookbanktv the one gift covid19 has give me i...</td>\n      <td>brookbanktv one gift covid19 give appreciation...</td>\n      <td>covid19</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>DIPR-J&amp;K</td>\n      <td>Jammu and Kashmir</td>\n      <td>🖊️Official Twitter handle of Department of Inf...</td>\n      <td>2017-02-12 06:45:15</td>\n      <td>101009</td>\n      <td>168</td>\n      <td>101</td>\n      <td>False</td>\n      <td>2020-07-25 12:27:08</td>\n      <td>25 July  Media Bulletin on Novel CoronaVirusUp...</td>\n      <td>['CoronaVirusUpdates', 'COVID19']</td>\n      <td>Twitter for Android</td>\n      <td>False</td>\n      <td>@DrSyedSehrish, @airnewsalerts, @ANI…</td>\n      <td>25 july media bulletin on novel coronavirusupd...</td>\n      <td>25 july media bulletin novel coronavirusupdate...</td>\n      <td>coronavirusupdates, covid19</td>\n      <td>2.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00005-96348dc3-b1c6-4c47-8a29-c29d961f5a8e","output_cleared":false,"source_hash":"7257a1aa","execution_millis":114,"execution_start":1602090225614},"source":"data = data.sample(n = 10000)\ndata.reset_index(inplace=True)\ndata.drop(['index', 'source'], axis = 1, inplace=True)\nprint(data.shape)\ndata.head(3)","execution_count":null,"outputs":[{"name":"stdout","text":"(25000, 17)\n","output_type":"stream"},{"output_type":"execute_result","execution_count":15,"data":{"application/vnd.deepnote.dataframe.v2+json":{"row_count":3,"column_count":17,"columns":[{"name":"user_name","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"Ageing Well B&H","count":1},{"name":"The Hindu - Chennai","count":1},{"name":"#COVID19: Stay at home","count":1}]}},{"name":"user_location","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"Brighton & Hove","count":1},{"name":"Chennai, India","count":1},{"name":"Accra, Ghana","count":1}]}},{"name":"user_description","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"We help people aged 50+ in Brighton & Hove find activities, groups, trips, befriending, and information in order to get the best from life.\nT:  07770 061 072","count":1},{"name":"The official twitter account of The Hindu's reporters in Chennai and across Tamil Nadu. Follow us for breaking news about the city and the State.","count":1},{"name":"Fact-checker | Journalist @Citi973 covering environment, climate change and social dev't | Past: @univers1057fm | \nhttps://t.co/c0ycXx7vq9","count":1}]}},{"name":"user_created","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"2019-08-21 09:38:48","count":1},{"name":"2012-06-20 11:24:09","count":1},{"name":"2010-08-22 16:49:27","count":1}]}},{"name":"user_followers","dtype":"int64","stats":{"unique_count":3,"nan_count":0,"min":267,"max":90393,"histogram":[{"bin_start":267,"bin_end":9279.6,"count":2},{"bin_start":9279.6,"bin_end":18292.2,"count":0},{"bin_start":18292.2,"bin_end":27304.800000000003,"count":0},{"bin_start":27304.800000000003,"bin_end":36317.4,"count":0},{"bin_start":36317.4,"bin_end":45330,"count":0},{"bin_start":45330,"bin_end":54342.600000000006,"count":0},{"bin_start":54342.600000000006,"bin_end":63355.200000000004,"count":0},{"bin_start":63355.200000000004,"bin_end":72367.8,"count":0},{"bin_start":72367.8,"bin_end":81380.40000000001,"count":0},{"bin_start":81380.40000000001,"bin_end":90393,"count":1}]}},{"name":"user_friends","dtype":"int64","stats":{"unique_count":3,"nan_count":0,"min":307,"max":623,"histogram":[{"bin_start":307,"bin_end":338.6,"count":1},{"bin_start":338.6,"bin_end":370.2,"count":1},{"bin_start":370.2,"bin_end":401.8,"count":0},{"bin_start":401.8,"bin_end":433.4,"count":0},{"bin_start":433.4,"bin_end":465,"count":0},{"bin_start":465,"bin_end":496.6,"count":0},{"bin_start":496.6,"bin_end":528.2,"count":0},{"bin_start":528.2,"bin_end":559.8,"count":0},{"bin_start":559.8,"bin_end":591.4000000000001,"count":0},{"bin_start":591.4000000000001,"bin_end":623,"count":1}]}},{"name":"user_favourites","dtype":"int64","stats":{"unique_count":3,"nan_count":0,"min":269,"max":6951,"histogram":[{"bin_start":269,"bin_end":937.2,"count":2},{"bin_start":937.2,"bin_end":1605.4,"count":0},{"bin_start":1605.4,"bin_end":2273.6000000000004,"count":0},{"bin_start":2273.6000000000004,"bin_end":2941.8,"count":0},{"bin_start":2941.8,"bin_end":3610,"count":0},{"bin_start":3610,"bin_end":4278.200000000001,"count":0},{"bin_start":4278.200000000001,"bin_end":4946.400000000001,"count":0},{"bin_start":4946.400000000001,"bin_end":5614.6,"count":0},{"bin_start":5614.6,"bin_end":6282.8,"count":0},{"bin_start":6282.8,"bin_end":6951,"count":1}]}},{"name":"user_verified","dtype":"bool","stats":{"unique_count":2,"nan_count":0,"categories":[{"name":"False","count":2},{"name":"True","count":1}]}},{"name":"date","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"2020-08-11 07:30:27","count":1},{"name":"2020-07-28 06:28:28","count":1},{"name":"2020-07-27 06:52:33","count":1}]}},{"name":"text","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"So much fun to be hard Dont miss out Email paulacarterimpactinitiativesorguk Brighton Hove zoom ","count":1},{"name":"At least 25 remand prisoners lodged in Perurani District Jail in Thoothukudi district have tested positive for ","count":1},{"name":"Talk to a psychologist today\n\nCOVID19 coronavirus COVIDー19 ","count":1}]}},{"name":"hashtags","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"['Brighton', 'Hove', 'zoom']","count":1},{"name":"['Thoothukudi']","count":1},{"name":"['COVID19', 'coronavirus', 'COVIDー19']","count":1}]}},{"name":"is_retweet","dtype":"bool","stats":{"unique_count":1,"nan_count":0,"categories":[{"name":"False","count":3}]}},{"name":"mention","dtype":"object","stats":{"unique_count":1,"nan_count":0,"categories":[{"name":"","count":3}]}},{"name":"english_text","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"so much fun to be hard dont miss out email paulacarterimpactinitiativesorguk brighton hove zoom ","count":1},{"name":"at least 25 remand prisoners lodged in perurani district jail in thoothukudi district have tested positive for ","count":1},{"name":"talk to a psychologist today covid19 coronavirus ","count":1}]}},{"name":"cleaned_text","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"much fun hard dont miss email paulacarterimpactinitiativesorguk brighton hove zoom ","count":1},{"name":"least 25 remand prisoners lodged perurani district jail thoothukudi district tested positive ","count":1},{"name":"talk psychologist today covid19 coronavirus ","count":1}]}},{"name":"cleaned_tags","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"brighton, hove, zoom","count":1},{"name":"thoothukudi","count":1},{"name":"covid19, coronavirus, covidー19","count":1}]}},{"name":"len_hashtag","dtype":"float64","stats":{"unique_count":2,"nan_count":0,"min":1,"max":3,"histogram":[{"bin_start":1,"bin_end":1.2,"count":1},{"bin_start":1.2,"bin_end":1.4,"count":0},{"bin_start":1.4,"bin_end":1.6,"count":0},{"bin_start":1.6,"bin_end":1.8,"count":0},{"bin_start":1.8,"bin_end":2,"count":0},{"bin_start":2,"bin_end":2.2,"count":0},{"bin_start":2.2,"bin_end":2.4000000000000004,"count":0},{"bin_start":2.4000000000000004,"bin_end":2.6,"count":0},{"bin_start":2.6,"bin_end":2.8,"count":0},{"bin_start":2.8,"bin_end":3,"count":2}]}},{"name":"_deepnote_index_column","dtype":"int64"}],"rows_top":[{"user_name":"Ageing Well B&H","user_location":"Brighton & Hove","user_description":"We help people aged 50+ in Brighton & Hove find activities, groups, trips, befriending, and information in order to get the best from life.\nT:  07770 061 072","user_created":"2019-08-21 09:38:48","user_followers":267,"user_friends":347,"user_favourites":569,"user_verified":false,"date":"2020-08-11 07:30:27","text":"So much fun to be hard Dont miss out Email paulacarterimpactinitiativesorguk Brighton Hove zoom ","hashtags":"['Brighton', 'Hove', 'zoom']","is_retweet":false,"mention":"","english_text":"so much fun to be hard dont miss out email paulacarterimpactinitiativesorguk brighton hove zoom ","cleaned_text":"much fun hard dont miss email paulacarterimpactinitiativesorguk brighton hove zoom ","cleaned_tags":"brighton, hove, zoom","len_hashtag":3,"_deepnote_index_column":0},{"user_name":"The Hindu - Chennai","user_location":"Chennai, India","user_description":"The official twitter account of The Hindu's reporters in Chennai and across Tamil Nadu. Follow us for breaking news about the city and the State.","user_created":"2012-06-20 11:24:09","user_followers":90393,"user_friends":307,"user_favourites":269,"user_verified":true,"date":"2020-07-28 06:28:28","text":"At least 25 remand prisoners lodged in Perurani District Jail in Thoothukudi district have tested positive for ","hashtags":"['Thoothukudi']","is_retweet":false,"mention":"","english_text":"at least 25 remand prisoners lodged in perurani district jail in thoothukudi district have tested positive for ","cleaned_text":"least 25 remand prisoners lodged perurani district jail thoothukudi district tested positive ","cleaned_tags":"thoothukudi","len_hashtag":1,"_deepnote_index_column":1},{"user_name":"#COVID19: Stay at home","user_location":"Accra, Ghana","user_description":"Fact-checker | Journalist @Citi973 covering environment, climate change and social dev't | Past: @univers1057fm | \nhttps://t.co/c0ycXx7vq9","user_created":"2010-08-22 16:49:27","user_followers":3103,"user_friends":623,"user_favourites":6951,"user_verified":false,"date":"2020-07-27 06:52:33","text":"Talk to a psychologist today\n\nCOVID19 coronavirus COVIDー19 ","hashtags":"['COVID19', 'coronavirus', 'COVIDー19']","is_retweet":false,"mention":"","english_text":"talk to a psychologist today covid19 coronavirus ","cleaned_text":"talk psychologist today covid19 coronavirus ","cleaned_tags":"covid19, coronavirus, covidー19","len_hashtag":3,"_deepnote_index_column":2}],"rows_bottom":null},"text/plain":"                user_name    user_location  \\\n0         Ageing Well B&H  Brighton & Hove   \n1     The Hindu - Chennai   Chennai, India   \n2  #COVID19: Stay at home     Accra, Ghana   \n\n                                    user_description         user_created  \\\n0  We help people aged 50+ in Brighton & Hove fin...  2019-08-21 09:38:48   \n1  The official twitter account of The Hindu's re...  2012-06-20 11:24:09   \n2  Fact-checker | Journalist @Citi973 covering en...  2010-08-22 16:49:27   \n\n   user_followers  user_friends  user_favourites  user_verified  \\\n0             267           347              569          False   \n1           90393           307              269           True   \n2            3103           623             6951          False   \n\n                  date                                               text  \\\n0  2020-08-11 07:30:27  So much fun to be hard Dont miss out Email pau...   \n1  2020-07-28 06:28:28  At least 25 remand prisoners lodged in Peruran...   \n2  2020-07-27 06:52:33  Talk to a psychologist today\\n\\nCOVID19 corona...   \n\n                                 hashtags  is_retweet mention  \\\n0            ['Brighton', 'Hove', 'zoom']       False           \n1                         ['Thoothukudi']       False           \n2  ['COVID19', 'coronavirus', 'COVIDー19']       False           \n\n                                        english_text  \\\n0  so much fun to be hard dont miss out email pau...   \n1  at least 25 remand prisoners lodged in peruran...   \n2  talk to a psychologist today covid19 coronavirus    \n\n                                        cleaned_text  \\\n0  much fun hard dont miss email paulacarterimpac...   \n1  least 25 remand prisoners lodged perurani dist...   \n2       talk psychologist today covid19 coronavirus    \n\n                     cleaned_tags  len_hashtag  \n0            brighton, hove, zoom          3.0  \n1                     thoothukudi          1.0  \n2  covid19, coronavirus, covidー19          3.0  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_name</th>\n      <th>user_location</th>\n      <th>user_description</th>\n      <th>user_created</th>\n      <th>user_followers</th>\n      <th>user_friends</th>\n      <th>user_favourites</th>\n      <th>user_verified</th>\n      <th>date</th>\n      <th>text</th>\n      <th>hashtags</th>\n      <th>is_retweet</th>\n      <th>mention</th>\n      <th>english_text</th>\n      <th>cleaned_text</th>\n      <th>cleaned_tags</th>\n      <th>len_hashtag</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Ageing Well B&amp;H</td>\n      <td>Brighton &amp; Hove</td>\n      <td>We help people aged 50+ in Brighton &amp; Hove fin...</td>\n      <td>2019-08-21 09:38:48</td>\n      <td>267</td>\n      <td>347</td>\n      <td>569</td>\n      <td>False</td>\n      <td>2020-08-11 07:30:27</td>\n      <td>So much fun to be hard Dont miss out Email pau...</td>\n      <td>['Brighton', 'Hove', 'zoom']</td>\n      <td>False</td>\n      <td></td>\n      <td>so much fun to be hard dont miss out email pau...</td>\n      <td>much fun hard dont miss email paulacarterimpac...</td>\n      <td>brighton, hove, zoom</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>The Hindu - Chennai</td>\n      <td>Chennai, India</td>\n      <td>The official twitter account of The Hindu's re...</td>\n      <td>2012-06-20 11:24:09</td>\n      <td>90393</td>\n      <td>307</td>\n      <td>269</td>\n      <td>True</td>\n      <td>2020-07-28 06:28:28</td>\n      <td>At least 25 remand prisoners lodged in Peruran...</td>\n      <td>['Thoothukudi']</td>\n      <td>False</td>\n      <td></td>\n      <td>at least 25 remand prisoners lodged in peruran...</td>\n      <td>least 25 remand prisoners lodged perurani dist...</td>\n      <td>thoothukudi</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>#COVID19: Stay at home</td>\n      <td>Accra, Ghana</td>\n      <td>Fact-checker | Journalist @Citi973 covering en...</td>\n      <td>2010-08-22 16:49:27</td>\n      <td>3103</td>\n      <td>623</td>\n      <td>6951</td>\n      <td>False</td>\n      <td>2020-07-27 06:52:33</td>\n      <td>Talk to a psychologist today\\n\\nCOVID19 corona...</td>\n      <td>['COVID19', 'coronavirus', 'COVIDー19']</td>\n      <td>False</td>\n      <td></td>\n      <td>talk to a psychologist today covid19 coronavirus</td>\n      <td>talk psychologist today covid19 coronavirus</td>\n      <td>covid19, coronavirus, covidー19</td>\n      <td>3.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00004-402a7d5e-a40d-4111-83f3-6c5a55be240b","output_cleared":false,"source_hash":"878de0e4","execution_millis":13,"execution_start":1602090225781},"source":"data.isnull().sum()","execution_count":null,"outputs":[{"output_type":"execute_result","execution_count":16,"data":{"text/plain":"user_name           0\nuser_location       0\nuser_description    0\nuser_created        0\nuser_followers      0\nuser_friends        0\nuser_favourites     0\nuser_verified       0\ndate                0\ntext                0\nhashtags            0\nis_retweet          0\nmention             0\nenglish_text        0\ncleaned_text        0\ncleaned_tags        0\nlen_hashtag         0\ndtype: int64"},"metadata":{}}]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00007-87a1cb7d-edca-4090-9983-fed1329c678b","output_cleared":false,"source_hash":"68c0e448","execution_millis":334455,"execution_start":1602090225798},"source":"from langdetect import detect\n\ndef detect_lang(txt):\n  try:\n    return detect(txt)\n  except:\n    return np.nan\n\ndata[\"language\"] = data.cleaned_text.apply(detect_lang)","execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00009-f7635db7-0dfc-4993-9bb9-d9e8eb426c93","output_cleared":false,"source_hash":"3361d07f","execution_millis":17,"execution_start":1602090560294},"source":"new_data = data[data.language == \"en\"]\nnew_data.reset_index(inplace = True)","execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00019-6ac7f537-fd89-4f89-b5a3-8b03793fd3d7","output_cleared":false,"source_hash":"5e19ce96","execution_start":1602090560353,"execution_millis":71},"source":"new_data.head(3)","execution_count":null,"outputs":[{"output_type":"execute_result","execution_count":19,"data":{"application/vnd.deepnote.dataframe.v2+json":{"row_count":3,"column_count":19,"columns":[{"name":"index","dtype":"int64","stats":{"unique_count":3,"nan_count":0,"min":0,"max":2,"histogram":[{"bin_start":0,"bin_end":0.2,"count":1},{"bin_start":0.2,"bin_end":0.4,"count":0},{"bin_start":0.4,"bin_end":0.6000000000000001,"count":0},{"bin_start":0.6000000000000001,"bin_end":0.8,"count":0},{"bin_start":0.8,"bin_end":1,"count":0},{"bin_start":1,"bin_end":1.2000000000000002,"count":1},{"bin_start":1.2000000000000002,"bin_end":1.4000000000000001,"count":0},{"bin_start":1.4000000000000001,"bin_end":1.6,"count":0},{"bin_start":1.6,"bin_end":1.8,"count":0},{"bin_start":1.8,"bin_end":2,"count":1}]}},{"name":"user_name","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"Ageing Well B&H","count":1},{"name":"The Hindu - Chennai","count":1},{"name":"#COVID19: Stay at home","count":1}]}},{"name":"user_location","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"Brighton & Hove","count":1},{"name":"Chennai, India","count":1},{"name":"Accra, Ghana","count":1}]}},{"name":"user_description","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"We help people aged 50+ in Brighton & Hove find activities, groups, trips, befriending, and information in order to get the best from life.\nT:  07770 061 072","count":1},{"name":"The official twitter account of The Hindu's reporters in Chennai and across Tamil Nadu. Follow us for breaking news about the city and the State.","count":1},{"name":"Fact-checker | Journalist @Citi973 covering environment, climate change and social dev't | Past: @univers1057fm | \nhttps://t.co/c0ycXx7vq9","count":1}]}},{"name":"user_created","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"2019-08-21 09:38:48","count":1},{"name":"2012-06-20 11:24:09","count":1},{"name":"2010-08-22 16:49:27","count":1}]}},{"name":"user_followers","dtype":"int64","stats":{"unique_count":3,"nan_count":0,"min":267,"max":90393,"histogram":[{"bin_start":267,"bin_end":9279.6,"count":2},{"bin_start":9279.6,"bin_end":18292.2,"count":0},{"bin_start":18292.2,"bin_end":27304.800000000003,"count":0},{"bin_start":27304.800000000003,"bin_end":36317.4,"count":0},{"bin_start":36317.4,"bin_end":45330,"count":0},{"bin_start":45330,"bin_end":54342.600000000006,"count":0},{"bin_start":54342.600000000006,"bin_end":63355.200000000004,"count":0},{"bin_start":63355.200000000004,"bin_end":72367.8,"count":0},{"bin_start":72367.8,"bin_end":81380.40000000001,"count":0},{"bin_start":81380.40000000001,"bin_end":90393,"count":1}]}},{"name":"user_friends","dtype":"int64","stats":{"unique_count":3,"nan_count":0,"min":307,"max":623,"histogram":[{"bin_start":307,"bin_end":338.6,"count":1},{"bin_start":338.6,"bin_end":370.2,"count":1},{"bin_start":370.2,"bin_end":401.8,"count":0},{"bin_start":401.8,"bin_end":433.4,"count":0},{"bin_start":433.4,"bin_end":465,"count":0},{"bin_start":465,"bin_end":496.6,"count":0},{"bin_start":496.6,"bin_end":528.2,"count":0},{"bin_start":528.2,"bin_end":559.8,"count":0},{"bin_start":559.8,"bin_end":591.4000000000001,"count":0},{"bin_start":591.4000000000001,"bin_end":623,"count":1}]}},{"name":"user_favourites","dtype":"int64","stats":{"unique_count":3,"nan_count":0,"min":269,"max":6951,"histogram":[{"bin_start":269,"bin_end":937.2,"count":2},{"bin_start":937.2,"bin_end":1605.4,"count":0},{"bin_start":1605.4,"bin_end":2273.6000000000004,"count":0},{"bin_start":2273.6000000000004,"bin_end":2941.8,"count":0},{"bin_start":2941.8,"bin_end":3610,"count":0},{"bin_start":3610,"bin_end":4278.200000000001,"count":0},{"bin_start":4278.200000000001,"bin_end":4946.400000000001,"count":0},{"bin_start":4946.400000000001,"bin_end":5614.6,"count":0},{"bin_start":5614.6,"bin_end":6282.8,"count":0},{"bin_start":6282.8,"bin_end":6951,"count":1}]}},{"name":"user_verified","dtype":"bool","stats":{"unique_count":2,"nan_count":0,"categories":[{"name":"False","count":2},{"name":"True","count":1}]}},{"name":"date","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"2020-08-11 07:30:27","count":1},{"name":"2020-07-28 06:28:28","count":1},{"name":"2020-07-27 06:52:33","count":1}]}},{"name":"text","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"So much fun to be hard Dont miss out Email paulacarterimpactinitiativesorguk Brighton Hove zoom ","count":1},{"name":"At least 25 remand prisoners lodged in Perurani District Jail in Thoothukudi district have tested positive for ","count":1},{"name":"Talk to a psychologist today\n\nCOVID19 coronavirus COVIDー19 ","count":1}]}},{"name":"hashtags","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"['Brighton', 'Hove', 'zoom']","count":1},{"name":"['Thoothukudi']","count":1},{"name":"['COVID19', 'coronavirus', 'COVIDー19']","count":1}]}},{"name":"is_retweet","dtype":"bool","stats":{"unique_count":1,"nan_count":0,"categories":[{"name":"False","count":3}]}},{"name":"mention","dtype":"object","stats":{"unique_count":1,"nan_count":0,"categories":[{"name":"","count":3}]}},{"name":"english_text","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"so much fun to be hard dont miss out email paulacarterimpactinitiativesorguk brighton hove zoom ","count":1},{"name":"at least 25 remand prisoners lodged in perurani district jail in thoothukudi district have tested positive for ","count":1},{"name":"talk to a psychologist today covid19 coronavirus ","count":1}]}},{"name":"cleaned_text","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"much fun hard dont miss email paulacarterimpactinitiativesorguk brighton hove zoom ","count":1},{"name":"least 25 remand prisoners lodged perurani district jail thoothukudi district tested positive ","count":1},{"name":"talk psychologist today covid19 coronavirus ","count":1}]}},{"name":"cleaned_tags","dtype":"object","stats":{"unique_count":3,"nan_count":0,"categories":[{"name":"brighton, hove, zoom","count":1},{"name":"thoothukudi","count":1},{"name":"covid19, coronavirus, covidー19","count":1}]}},{"name":"len_hashtag","dtype":"float64","stats":{"unique_count":2,"nan_count":0,"min":1,"max":3,"histogram":[{"bin_start":1,"bin_end":1.2,"count":1},{"bin_start":1.2,"bin_end":1.4,"count":0},{"bin_start":1.4,"bin_end":1.6,"count":0},{"bin_start":1.6,"bin_end":1.8,"count":0},{"bin_start":1.8,"bin_end":2,"count":0},{"bin_start":2,"bin_end":2.2,"count":0},{"bin_start":2.2,"bin_end":2.4000000000000004,"count":0},{"bin_start":2.4000000000000004,"bin_end":2.6,"count":0},{"bin_start":2.6,"bin_end":2.8,"count":0},{"bin_start":2.8,"bin_end":3,"count":2}]}},{"name":"language","dtype":"object","stats":{"unique_count":1,"nan_count":0,"categories":[{"name":"en","count":3}]}},{"name":"_deepnote_index_column","dtype":"int64"}],"rows_top":[{"index":0,"user_name":"Ageing Well B&H","user_location":"Brighton & Hove","user_description":"We help people aged 50+ in Brighton & Hove find activities, groups, trips, befriending, and information in order to get the best from life.\nT:  07770 061 072","user_created":"2019-08-21 09:38:48","user_followers":267,"user_friends":347,"user_favourites":569,"user_verified":false,"date":"2020-08-11 07:30:27","text":"So much fun to be hard Dont miss out Email paulacarterimpactinitiativesorguk Brighton Hove zoom ","hashtags":"['Brighton', 'Hove', 'zoom']","is_retweet":false,"mention":"","english_text":"so much fun to be hard dont miss out email paulacarterimpactinitiativesorguk brighton hove zoom ","cleaned_text":"much fun hard dont miss email paulacarterimpactinitiativesorguk brighton hove zoom ","cleaned_tags":"brighton, hove, zoom","len_hashtag":3,"language":"en","_deepnote_index_column":0},{"index":1,"user_name":"The Hindu - Chennai","user_location":"Chennai, India","user_description":"The official twitter account of The Hindu's reporters in Chennai and across Tamil Nadu. Follow us for breaking news about the city and the State.","user_created":"2012-06-20 11:24:09","user_followers":90393,"user_friends":307,"user_favourites":269,"user_verified":true,"date":"2020-07-28 06:28:28","text":"At least 25 remand prisoners lodged in Perurani District Jail in Thoothukudi district have tested positive for ","hashtags":"['Thoothukudi']","is_retweet":false,"mention":"","english_text":"at least 25 remand prisoners lodged in perurani district jail in thoothukudi district have tested positive for ","cleaned_text":"least 25 remand prisoners lodged perurani district jail thoothukudi district tested positive ","cleaned_tags":"thoothukudi","len_hashtag":1,"language":"en","_deepnote_index_column":1},{"index":2,"user_name":"#COVID19: Stay at home","user_location":"Accra, Ghana","user_description":"Fact-checker | Journalist @Citi973 covering environment, climate change and social dev't | Past: @univers1057fm | \nhttps://t.co/c0ycXx7vq9","user_created":"2010-08-22 16:49:27","user_followers":3103,"user_friends":623,"user_favourites":6951,"user_verified":false,"date":"2020-07-27 06:52:33","text":"Talk to a psychologist today\n\nCOVID19 coronavirus COVIDー19 ","hashtags":"['COVID19', 'coronavirus', 'COVIDー19']","is_retweet":false,"mention":"","english_text":"talk to a psychologist today covid19 coronavirus ","cleaned_text":"talk psychologist today covid19 coronavirus ","cleaned_tags":"covid19, coronavirus, covidー19","len_hashtag":3,"language":"en","_deepnote_index_column":2}],"rows_bottom":null},"text/plain":"   index               user_name    user_location  \\\n0      0         Ageing Well B&H  Brighton & Hove   \n1      1     The Hindu - Chennai   Chennai, India   \n2      2  #COVID19: Stay at home     Accra, Ghana   \n\n                                    user_description         user_created  \\\n0  We help people aged 50+ in Brighton & Hove fin...  2019-08-21 09:38:48   \n1  The official twitter account of The Hindu's re...  2012-06-20 11:24:09   \n2  Fact-checker | Journalist @Citi973 covering en...  2010-08-22 16:49:27   \n\n   user_followers  user_friends  user_favourites  user_verified  \\\n0             267           347              569          False   \n1           90393           307              269           True   \n2            3103           623             6951          False   \n\n                  date                                               text  \\\n0  2020-08-11 07:30:27  So much fun to be hard Dont miss out Email pau...   \n1  2020-07-28 06:28:28  At least 25 remand prisoners lodged in Peruran...   \n2  2020-07-27 06:52:33  Talk to a psychologist today\\n\\nCOVID19 corona...   \n\n                                 hashtags  is_retweet mention  \\\n0            ['Brighton', 'Hove', 'zoom']       False           \n1                         ['Thoothukudi']       False           \n2  ['COVID19', 'coronavirus', 'COVIDー19']       False           \n\n                                        english_text  \\\n0  so much fun to be hard dont miss out email pau...   \n1  at least 25 remand prisoners lodged in peruran...   \n2  talk to a psychologist today covid19 coronavirus    \n\n                                        cleaned_text  \\\n0  much fun hard dont miss email paulacarterimpac...   \n1  least 25 remand prisoners lodged perurani dist...   \n2       talk psychologist today covid19 coronavirus    \n\n                     cleaned_tags  len_hashtag language  \n0            brighton, hove, zoom          3.0       en  \n1                     thoothukudi          1.0       en  \n2  covid19, coronavirus, covidー19          3.0       en  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>index</th>\n      <th>user_name</th>\n      <th>user_location</th>\n      <th>user_description</th>\n      <th>user_created</th>\n      <th>user_followers</th>\n      <th>user_friends</th>\n      <th>user_favourites</th>\n      <th>user_verified</th>\n      <th>date</th>\n      <th>text</th>\n      <th>hashtags</th>\n      <th>is_retweet</th>\n      <th>mention</th>\n      <th>english_text</th>\n      <th>cleaned_text</th>\n      <th>cleaned_tags</th>\n      <th>len_hashtag</th>\n      <th>language</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>Ageing Well B&amp;H</td>\n      <td>Brighton &amp; Hove</td>\n      <td>We help people aged 50+ in Brighton &amp; Hove fin...</td>\n      <td>2019-08-21 09:38:48</td>\n      <td>267</td>\n      <td>347</td>\n      <td>569</td>\n      <td>False</td>\n      <td>2020-08-11 07:30:27</td>\n      <td>So much fun to be hard Dont miss out Email pau...</td>\n      <td>['Brighton', 'Hove', 'zoom']</td>\n      <td>False</td>\n      <td></td>\n      <td>so much fun to be hard dont miss out email pau...</td>\n      <td>much fun hard dont miss email paulacarterimpac...</td>\n      <td>brighton, hove, zoom</td>\n      <td>3.0</td>\n      <td>en</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>The Hindu - Chennai</td>\n      <td>Chennai, India</td>\n      <td>The official twitter account of The Hindu's re...</td>\n      <td>2012-06-20 11:24:09</td>\n      <td>90393</td>\n      <td>307</td>\n      <td>269</td>\n      <td>True</td>\n      <td>2020-07-28 06:28:28</td>\n      <td>At least 25 remand prisoners lodged in Peruran...</td>\n      <td>['Thoothukudi']</td>\n      <td>False</td>\n      <td></td>\n      <td>at least 25 remand prisoners lodged in peruran...</td>\n      <td>least 25 remand prisoners lodged perurani dist...</td>\n      <td>thoothukudi</td>\n      <td>1.0</td>\n      <td>en</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>#COVID19: Stay at home</td>\n      <td>Accra, Ghana</td>\n      <td>Fact-checker | Journalist @Citi973 covering en...</td>\n      <td>2010-08-22 16:49:27</td>\n      <td>3103</td>\n      <td>623</td>\n      <td>6951</td>\n      <td>False</td>\n      <td>2020-07-27 06:52:33</td>\n      <td>Talk to a psychologist today\\n\\nCOVID19 corona...</td>\n      <td>['COVID19', 'coronavirus', 'COVIDー19']</td>\n      <td>False</td>\n      <td></td>\n      <td>talk to a psychologist today covid19 coronavirus</td>\n      <td>talk psychologist today covid19 coronavirus</td>\n      <td>covid19, coronavirus, covidー19</td>\n      <td>3.0</td>\n      <td>en</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00021-c2c9f0a7-7245-4c15-a7ad-8c94048c7c74","output_cleared":false,"source_hash":"ac4c14a4","execution_millis":19,"execution_start":1602090560428},"source":"new_data.drop(['index'], axis=1, inplace=True)","execution_count":null,"outputs":[{"name":"stderr","text":"/opt/venv/lib/python3.7/site-packages/pandas/core/frame.py:3997: SettingWithCopyWarning: \nA value is trying to be set on a copy of a slice from a DataFrame\n\nSee the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n  errors=errors,\n","output_type":"stream"}]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00022-10432ac5-6883-496d-b66f-1c560eed5c55","output_cleared":false,"source_hash":"9c624f2a","execution_millis":963,"execution_start":1602090560454},"source":"new_data.to_csv('english_tweets.csv')","execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"tags":[],"cell_id":"00023-0cd683e4-fc1d-43d0-9da2-7b67901eeeba"},"source":"","execution_count":null,"outputs":[]}],"nbformat":4,"nbformat_minor":2,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.3"},"deepnote_notebook_id":"954ccfaa-a034-4082-8375-23ec57c55910","deepnote_execution_queue":[]}}