diff --git a/scripts/twitter-api-example/.ipynb_checkpoints/api-twitter-Group7-checkpoint.ipynb b/scripts/twitter-api-example/.ipynb_checkpoints/api-twitter-Group7-checkpoint.ipynb new file mode 100644 index 0000000..b7ccf86 --- /dev/null +++ b/scripts/twitter-api-example/.ipynb_checkpoints/api-twitter-Group7-checkpoint.ipynb @@ -0,0 +1,817 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "####################\n", + "#Author: brandon chiazza\n", + "#version: 1.0\n", + "#purpose: to call a twitter api and return results\n", + "#documentation: https://developer.twitter.com/en/docs\n", + "#####################\n", + "\n", + "import pandas as pd\n", + "import requests\n", + "import json\n", + "import base64\n", + "#!pip install s3fs\n", + "import s3fs # documentation: https://s3fs.readthedocs.io/en/latest/\n", + "import time\n", + "import twitter_keys #this is a custom reference module to a package containing twitter keys\n", + "\n", + "%config IPCompleter.greedy=True\n", + "\n", + "key_secret = '{}:{}'.format(twitter_keys.client_key, twitter_keys.client_secret).encode('ascii')\n", + "#key_secret = '{}:{}'.format('123', twitter_keys.client_secret).encode('ascii')\n", + "b64_encoded_key = base64.b64encode(key_secret)\n", + "b64_encoded_key = b64_encoded_key.decode('ascii')\n", + "\n", + "#identify base url and oauth token path\n", + "base_url = 'https://api.twitter.com/' #base url for authentication\n", + "auth_url = '{}oauth2/token'.format(base_url)\n", + "\n", + "#share header information -- encoding is ascii\n", + "auth_headers = {\n", + " 'Authorization': 'Basic {}'.format(b64_encoded_key),\n", + " 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'\n", + "}\n", + "\n", + "#pass clientcredentials\n", + "auth_data = {\n", + " 'grant_type': 'client_credentials'\n", + "}\n", + "\n", + "#send authentication using requests - POST request\n", + "auth_resp = requests.post(auth_url, headers=auth_headers, data=auth_data)\n", + "\n", + "#check response status. 200 = OK\n", + "auth_resp.status_code\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['token_type', 'access_token'])\n" + ] + }, + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "#Keys in data response are token_type (bearer) and access_token (your access token)\n", + "print(auth_resp.json().keys())\n", + "\n", + "access_token = auth_resp.json()['access_token']\n", + "\n", + "\n", + "search_headers = {\n", + " 'Authorization': 'Bearer {}'.format(access_token) \n", + "}\n", + "\n", + "#enter search parameters for coronavirus example. This looks for \"covid-19\" in the 1000 most recent tweets\n", + "query_params = {\n", + " 'q': 'covid-19',\n", + " 'result_type': 'recent',\n", + " 'count': 100, #update here to get more/less than 1000 returns\n", + " 'lang': 'en' #filters by english language only\n", + "}\n", + "\n", + "\n", + "#identify search url path and save \n", + "search_url = '{}1.1/search/tweets.json'.format(base_url)\n", + "\n", + "\n", + "#run search using get request\n", + "search_resp = requests.get(search_url, headers=search_headers, params=query_params)\n", + "\n", + "#check status code of GET request\n", + "search_resp.status_code\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "RT @soompi: #NCT_DOJAEJUNG Cancels This Week’s Promotions After #Jaehyun Experiences COVID-19 Symptoms\n", + "https://t.co/eVKT4JtqBX https://t.co…\n", + "\n", + "RT @KanekoaTheGreat: Former DNI John Ratcliffe says the CDC Director, the Secretary of State, and the Director of National Intelligence agr…\n", + "\n", + "RT @sanemiliomps: @sanemiliomps Prevention on Covid-19 https://t.co/9NdduKkvKo\n", + "\n", + "RT @laurieallee: From 2020:\n", + "\"1/3of patients with COVID-19 ― both active and cleared cases ― showed at least some measure of myocarditis.\" h…\n", + "\n", + "RT @MakisMD: Vista, CA - 40 year old Pastor Jay Foulk died in his sleep on April 13, 2023.\n", + "\n", + "He was negative for COVID-19, went to bed feeli…\n", + "\n", + "RT @sanemiliomps: @sanemiliomps COVID-19 SAFETY TIPS https://t.co/v7FG6Civ4P\n", + "\n", + "RT @stkirsch: Despite the fact that it will increase your chance of getting Covid, FDA authorizes another shot for older adults and people…\n", + "\n", + "RT @laralogan: These people are either literally insane or pure evil or both?\n", + "\n", + "It’s one thing for ordinary citizens to still be blinded by…\n", + "\n", + "RT @RogerMarshallMD: I sat down with @ABC and @TerryMoran to share the Origin of COVID-19 Report and why it is so important so we ensure th…\n", + "\n", + "RT @ShashiTharoor: On this issue ⁦@DrSJaishankar’s response surprises me. He knows that public representatives frequently⁩ convey to Govt a…\n", + "\n", + "RT @US_FDA: The monovalent Moderna and Pfizer-BioNTech COVID-19 vaccines are no longer authorized for use in the United States.\n", + "\n", + "RT @MakisMD: 43 year old UK Police Officer Andy Boardman collapsed while responding to a call and died suddenly on April 11, 2023.\n", + "\n", + "He leav…\n", + "\n", + "RT @TonyHinton2016: Doctor's death due to AstraZeneca Covid vaccine reaction - inquest - BBC News\n", + "\n", + "\"The death of a doctor after his Oxford-…\n", + "\n", + "RT @kevinnbass: In today's House Appropriations Committee hearing, CDC Director Rochelle Walensky lied directly to Congress, repeating her…\n", + "\n", + "RT @CourthouseNews: A Ninth Circuit panel on Wednesday confirmed the legality of President Joe Biden's Covid-19 vaccine mandate for federal…\n", + "\n", + "As federal aid winds down, long COVID patients feel left behind : Shots - Health News \n", + "\n", + "https://t.co/O8KRLfquff\n", + "\n", + "RT @COVIDSelect: 🔥🔥🔥\n", + " \n", + "“This has the potential to be one of the greatest medical disasters in world history, but also one of the greatest c…\n", + "\n", + "@Bucks @Marjon32 Guys Run and Dunk actually has 0 real awards 😭\n", + "2018-19 MVP: Only got it because he played on a sup… https://t.co/nfLi7njygj\n", + "\n", + "RT @US_FDA: The monovalent Moderna and Pfizer-BioNTech COVID-19 vaccines are no longer authorized for use in the United States.\n", + "\n", + "RT @RepThomasMassie: 2020: Candidate Biden called President Trump xenophobic for having a COVID-19 travel ban on China.\n", + "\n", + "But now that the C…\n", + "\n", + "RT @STVASTG: Right now in America, it is easier to get an AR-15 than a test kit for COVID-19. #shooter #gunviolence #guncontrolnow #gunviol…\n", + "\n", + "RT @albertan_angry: The real headline here should be that Canadians have zero right to medical privacy. Whatever you say to your doctor ca…\n", + "\n", + "RT @EdKrassen: As alleged COVID conspiracy theorist RFK Jr. today officially announced his candidacy for President, anti-vaxxers are once a…\n", + "\n", + "Biden's COVID vaccine rule for federal contractors was valid, US court rules https://t.co/OoQQOLlyyv… https://t.co/BTtYRuDzjS\n", + "\n", + "With the “tripledemic” (Covid/RSV/Flu) on the rise the Tasmanian Covid@homeplus program is keeping me and many othe… https://t.co/UkRGBse2gM\n", + "\n", + "RT @OccupyDemocrats: BREAKING: Marjorie Taylor Greene is publicly humiliated at a hearing on the origins of COVID-19 as Dr. Mark Lowenthal…\n", + "\n", + "Iloilo City Mayor Jerry Treñas on Wednesday issued an advisory reiterating national government policy on the wearin… https://t.co/3imKF093wO\n", + "\n", + "RT @MarkNeugebaue13: Exclusive: Australian Doctor Suspended for Speaking Out Against COVID-19 Vaccines Takes Medical Regulators to High Cou…\n", + "\n", + "RT @zerohedge: Watch: Former Director Of National Intelligence Admits That Fauci Lied About Gain Of Function Research https://t.co/CbEryBar…\n", + "\n", + "RT @MorningsMaria: Jamie Metzl urges Biden admin to ‘keep digging’ on origins of COVID-19 https://t.co/p6Id7PpeQ8\n", + "\n", + "RT @nytimes: Breaking News: The CDC recommended that adults 65 and older and those with weakened immune systems get another dose of the ref…\n", + "\n", + "RT @laralogan: These people are either literally insane or pure evil or both?\n", + "\n", + "It’s one thing for ordinary citizens to still be blinded by…\n", + "\n", + "RT @robster12065612: YOUR GOVERNMENT LIKES YOU THAT MUCH THEY PUSHED AN INJECTION ON YOU KNOWING THE COVID 19 WAS NEVER GOING TO KILL YOU B…\n", + "\n", + "RT @US_FDA: The monovalent Moderna and Pfizer-BioNTech COVID-19 vaccines are no longer authorized for use in the United States.\n", + "\n", + "RT @The_Trump_Train: WOW - The Moderna and Pfizer Covid 19 vaccines are NO LONGER AUTHORIZED for use in the United States. \n", + "\n", + "They must be f…\n", + "\n", + "RT @ZekeEmanuel: There is still no test, treatment, or cure for #LongCovid. There isn't even an accepted medical definition. We need to spe…\n", + "\n", + "RT @TonyHinton2016: Doctor's death due to AstraZeneca Covid vaccine reaction - inquest - BBC News\n", + "\n", + "\"The death of a doctor after his Oxford-…\n", + "\n", + "@elhoelho @therealrukshan So, @elhoelho did you know the @AstraZeneca COVID-19 “vaccine” product has been officiall… https://t.co/Fc1czrMa4E\n", + "\n", + "RT @kelcsimpkins: The CDC estimated in March that 6% of U.S. adults, or about 16 million, were experiencing #LongCovid, or ongoing health p…\n", + "\n", + "RT @Covid19Critical: \"The more shots you get, the greater your risk of getting COVID... so it has negative efficacy.\"\n", + "\n", + "Earlier this month,…\n", + "\n", + "RT @snopes: One popular social media post in the early days of the pandemic played on conspiracy theories by purporting to show a railroad…\n", + "\n", + "RT @Acyn: Moskowitz: I think the American people have a right to know why so much misinformation was spread about Covid 19 in this country…\n", + "\n", + "RT @kevinnbass: In today's House Appropriations Committee hearing, CDC Director Rochelle Walensky lied directly to Congress, repeating her…\n", + "\n", + "RT @DrGorfinkel: Think the pandemic’s over?\n", + "\n", + "COVID-19 is 🇨🇦’s 3rd leading cause of death. https://t.co/lbOEIujk41\n", + "\n", + "@NicoleMirante @RobertKennedyJr “I'm living in hell...\" Arrhythmia, discomfort, deep vein phlebectomy, a whole stri… https://t.co/13UiY2VYVm\n", + "\n", + "RT @kevinnbass: In today's House Appropriations Committee hearing, CDC Director Rochelle Walensky lied directly to Congress, repeating her…\n", + "\n", + "RT @CKellyUAP: 18,782 COVID DEATHS SINCE THE TGA BANNED IVERMECTIN.\n", + "\n", + "The TGA banned Ivermectin on the 10th September 2021, giving the most…\n", + "\n", + "@seanhannity Fauci deserve the highest reward medal and certificate for his courageous effort in combating COVID-19… https://t.co/jWGLnlcBp6\n", + "\n", + "RT @VPrasadMDMPH: I don't know if he plagiarized but I do know he is a terrible Covid-19 reporter. Arrogant and incurious.\n", + "\n", + "RT @zerohedge: Watch: Former Director Of National Intelligence Admits That Fauci Lied About Gain Of Function Research https://t.co/CbEryBar…\n", + "\n", + "RT @zerohedge: Watch: Former Director Of National Intelligence Admits That Fauci Lied About Gain Of Function Research https://t.co/CbEryBar…\n", + "\n", + "@covid19nz Collectively, this society doesn't have the workforce, the financial support, or the COURAGE to Unite Ag… https://t.co/5Llj1wZyAT\n", + "\n", + "RT @KanekoaTheGreat: 10/ Chris Krebs political bias and his affinity for domestic censorship:\n", + "\n", + "•Called Trump a national security threat\n", + "•Pl…\n", + "\n", + "RT @aiesecinghana: Her evolvement in the awareness of COVID 19 has been tremendous, she also donates to support child care https://t.co/5…\n", + "\n", + "RT @EdKrassen: As alleged COVID conspiracy theorist RFK Jr. today officially announced his candidacy for President, anti-vaxxers are once a…\n", + "\n", + "SciTechDaily: An Unlikely Hero: How the Common Cold Shields Children From COVID-19 https://t.co/ZP50eTr9PP\n", + "\n", + "One popular social media post in the early days of the pandemic played on conspiracy theories by purporting to show… https://t.co/6lbY7B1l2P\n", + "\n", + "@realLizUSA This is why Donald Trump lost the election. \n", + "\n", + "He will never be President again.\n", + "\n", + "Damning analysis of Tr… https://t.co/3X4bTWmQec\n", + "\n", + "COVID-19 virus, the global disaster and the public enemy of the world, has been sheltered from the lonely dream to… https://t.co/4VYmjnKJg4\n", + "\n", + "RT @DrLiMengYAN1: Takeaways on #COVID19 origin @COVIDSelect \n", + "1. Lab Leak ‘Only Explanation Credibly Supported by Intelligence, Science & Co…\n", + "\n", + "RT @kevinnbass: In today's House Appropriations Committee hearing, CDC Director Rochelle Walensky lied directly to Congress, repeating her…\n", + "\n", + "RT @zerohedge: Watch: Former Director Of National Intelligence Admits That Fauci Lied About Gain Of Function Research https://t.co/CbEryBar…\n", + "\n", + "RT @kevinnbass: In today's House Appropriations Committee hearing, CDC Director Rochelle Walensky lied directly to Congress, repeating her…\n", + "\n", + "RT @zerohedge: Watch: Former Director Of National Intelligence Admits That Fauci Lied About Gain Of Function Research https://t.co/CbEryBar…\n", + "\n", + "@JoshMatlow If only these great spaces were safe. #covidisairborne Want to LEAD? Show the city you're going to work… https://t.co/EoYg5cxMzy\n", + "\n", + "RT @rdumont99: For those who've used BC's \"COVID-19 Dashboard\", it wasn't updated this week and next week will be the last\n", + "\n", + "https://t.co/gd…\n", + "\n", + "Out of these, 421 children were diagnosed with COVID-19, and 3,298 children did not have COVID-19, but both groups… https://t.co/M8fESQCSNh\n", + "\n", + "RT @KelleyKga: In today's ACIP, CDC cited this KFF survey that found most people who had gotten a bivalent thought it was important to get…\n", + "\n", + "RT @captainuwu: \"Hello.\n", + "\n", + "We would like to tell you updates regarding Jaehyun's health.\n", + "\n", + "Jaehyun visited the hospital and had a COVID-19 PCR…\n", + "\n", + "RT @CNN: The Biden administration unveiled a $1.1 billion program aimed at providing Covid-19 vaccines and treatments to the uninsured at n…\n", + "\n", + "RT @kevinnbass: In today's House Appropriations Committee hearing, CDC Director Rochelle Walensky lied directly to Congress, repeating her…\n", + "\n", + "RT @KanekoaTheGreat: Former DNI John Ratcliffe says the CDC Director, the Secretary of State, and the Director of National Intelligence agr…\n", + "\n", + "RT @nytimes: Breaking News: The CDC recommended that adults 65 and older and those with weakened immune systems get another dose of the ref…\n", + "\n", + "U.S. Authorizes a New Round of Covid Boosters https://t.co/hiBujni2Hs\n", + "\n", + "RT @c_s_wallace: ‘Wearing masks was common in Japan even before the COVID-19 pandemic, and some people regularly use masks during hay fever…\n", + "\n", + "RT @nytimes: Breaking News: The CDC recommended that adults 65 and older and those with weakened immune systems get another dose of the ref…\n", + "\n", + "RT @NFSC999: Zhang Yutong, a 5-year-old boy from China's Henan province, was diagnosed with acute leukemia after an examination for weaknes…\n", + "\n", + "RT @mtaibbi: 1.TWITTER FILES #19\n", + "The Great Covid-19 Lie Machine\n", + "Stanford, the Virality Project, and the Censorship of “True Stories” https:…\n", + "\n", + "RT @MakisMD: 43 year old UK Police Officer Andy Boardman collapsed while responding to a call and died suddenly on April 11, 2023.\n", + "\n", + "He leav…\n", + "\n", + "RT @B52Malmet: CDC greenlights spring COVID booster for some. Do you need it? I’m getting mine Monday. That’ll be six shots for me all toge…\n", + "\n", + "RT @MakisMD: Vista, CA - 40 year old Pastor Jay Foulk died in his sleep on April 13, 2023.\n", + "\n", + "He was negative for COVID-19, went to bed feeli…\n", + "\n", + "RT @JoshDehaas: In April 2023, doctors in Alberta have again denied Anette Lewis access to the transplant list because she won’t accept COV…\n", + "\n", + "RT @Lidsville: \"Hospitalizations for COVID-19 among Canada’s children & youth surged by more than 600% in the 2nd year of the pandemic\"\n", + "\n", + "Re…\n", + "\n", + "RT @LionAdvocacy: Today is April 18th, 2023.\n", + "\n", + "Today, the FDA announced the monovalent Pfizer & Moderna jabs are no longer authorized.\n", + "\n", + "Toda…\n", + "\n", + "RT @EdKrassen: As alleged COVID conspiracy theorist RFK Jr. today officially announced his candidacy for President, anti-vaxxers are once a…\n", + "\n", + "RT @stkirsch: Despite the fact that it will increase your chance of getting Covid, FDA authorizes another shot for older adults and people…\n", + "\n", + "RT @B52Malmet: CDC greenlights spring COVID booster for some. Do you need it? I’m getting mine Monday. That’ll be six shots for me all toge…\n", + "\n", + "#LongCovid NOT psychosomatic:\n", + "\n", + "”Our results should not be misinterpreted as supporting a hypothesis that post–COVID… https://t.co/5eeRK5vwhX\n", + "\n", + "RT @CKellyUAP: 18,782 COVID DEATHS SINCE THE TGA BANNED IVERMECTIN.\n", + "\n", + "The TGA banned Ivermectin on the 10th September 2021, giving the most…\n", + "\n", + "RT @TonyHinton2016: Doctor's death due to AstraZeneca Covid vaccine reaction - inquest - BBC News\n", + "\n", + "\"The death of a doctor after his Oxford-…\n", + "\n", + "RT @WebMD: New research is shedding light on how an infection with COVID-19 may reactivate, or even cause, psoriasis.\n", + "https://t.co/pZ6nNDi1…\n", + "\n", + "RT @HopeRising19: How much #nz $$ were spent on the Unite Against Covid 19 and the Vaccine promotion campaign?\n", + "\n", + "Just marketing \n", + "\n", + "From April…\n", + "\n", + "RT @DailyClout: Children's Health Defense: The U.S. government approved its first three payments to people injured by COVID-19 vaccines — a…\n", + "\n", + "RT @Teagan1776: Did you know?? Todays News that went under/unreported- The monovalent Moderna & Pfizer BioNTech Covid-19 “vaccines” are no…\n", + "\n", + "RT @Storiesofinjury: A 56 year old math teacher had an Intracranial Hemorrhage after the first dose of Pfizer in March of 2023. She unfort…\n", + "\n", + "RT @De_Spectator_: Ex Director of National Intelligence John Ratcliffe says there’s overwhelming evidence that COVID-19 was manufactured in…\n", + "\n", + "RT @McMasterU: COVID-19 is the third-leading cause of death in Canada, but it’s older people who are dying. That we accept this and carry o…\n", + "\n", + "RT @michaelpsenger: “A physician who fled communist Yugoslavia saw the COVID-19 pandemic ‘through two eyes’…lockdowns were imposed…‘That wa…\n", + "\n", + "RT @Roman_Baber: \"The death of a doctor after his Oxford-AstraZeneca Covid-19 jab was due to unintended complications of the vaccine, an in…\n", + "\n", + "RT @MackayIM: If 1% of COVID-19 cases result in death, does that mean you have a 1% chance of dying if you catch it? A mathematician explai…\n", + "\n" + ] + } + ], + "source": [ + "#print text from result to verify \n", + "twitter_data = search_resp.json()\n", + "\n", + "for x in twitter_data['statuses']:\n", + " print(x['text'] + '\\n')\n", + " #break #prints after one iteration and stops, remove break to see all 1000" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
created_atidid_strtexttruncatedentitiesmetadatasourcein_reply_to_status_idin_reply_to_status_id_str...retweet_countfavorite_countfavoritedretweetedpossibly_sensitivelangextended_entitiesquoted_status_idquoted_status_id_strquoted_status
0Thu Apr 20 00:56:50 +0000 202316488531929799557151648853192979955715RT @soompi: #NCT_DOJAEJUNG Cancels This Week’s...False{'hashtags': [{'text': 'NCT_DOJAEJUNG', 'indic...{'iso_language_code': 'en', 'result_type': 're...<a href=\"http://twitter.com/download/android\" ...NaNNone...2030FalseFalseFalseenNaNNaNNaNNaN
1Thu Apr 20 00:56:49 +0000 202316488531924894597121648853192489459712RT @KanekoaTheGreat: Former DNI John Ratcliffe...False{'hashtags': [], 'symbols': [], 'user_mentions...{'iso_language_code': 'en', 'result_type': 're...<a href=\"http://twitter.com/download/android\" ...NaNNone...170450FalseFalseNaNenNaNNaNNaNNaN
2Thu Apr 20 00:56:47 +0000 202316488531813242183681648853181324218368RT @sanemiliomps: @sanemiliomps Prevention on ...False{'hashtags': [], 'symbols': [], 'user_mentions...{'iso_language_code': 'en', 'result_type': 're...<a href=\"http://twitter.com/download/android\" ...NaNNone...20FalseFalseFalseen{'media': [{'id': 1648176605196603394, 'id_str...NaNNaNNaN
3Thu Apr 20 00:56:47 +0000 202316488531807789547541648853180778954754RT @laurieallee: From 2020:\\n\"1/3of patients w...False{'hashtags': [], 'symbols': [], 'user_mentions...{'iso_language_code': 'en', 'result_type': 're...<a href=\"http://twitter.com/download/android\" ...NaNNone...3000FalseFalseNaNenNaNNaNNaNNaN
4Thu Apr 20 00:56:46 +0000 202316488531780189306911648853178018930691RT @MakisMD: Vista, CA - 40 year old Pastor Ja...False{'hashtags': [], 'symbols': [], 'user_mentions...{'iso_language_code': 'en', 'result_type': 're...<a href=\"https://mobile.twitter.com\" rel=\"nofo...NaNNone...6970FalseFalseNaNenNaNNaNNaNNaN
\n", + "

5 rows × 30 columns

\n", + "
" + ], + "text/plain": [ + " created_at id id_str \\\n", + "0 Thu Apr 20 00:56:50 +0000 2023 1648853192979955715 1648853192979955715 \n", + "1 Thu Apr 20 00:56:49 +0000 2023 1648853192489459712 1648853192489459712 \n", + "2 Thu Apr 20 00:56:47 +0000 2023 1648853181324218368 1648853181324218368 \n", + "3 Thu Apr 20 00:56:47 +0000 2023 1648853180778954754 1648853180778954754 \n", + "4 Thu Apr 20 00:56:46 +0000 2023 1648853178018930691 1648853178018930691 \n", + "\n", + " text truncated \\\n", + "0 RT @soompi: #NCT_DOJAEJUNG Cancels This Week’s... False \n", + "1 RT @KanekoaTheGreat: Former DNI John Ratcliffe... False \n", + "2 RT @sanemiliomps: @sanemiliomps Prevention on ... False \n", + "3 RT @laurieallee: From 2020:\\n\"1/3of patients w... False \n", + "4 RT @MakisMD: Vista, CA - 40 year old Pastor Ja... False \n", + "\n", + " entities \\\n", + "0 {'hashtags': [{'text': 'NCT_DOJAEJUNG', 'indic... \n", + "1 {'hashtags': [], 'symbols': [], 'user_mentions... \n", + "2 {'hashtags': [], 'symbols': [], 'user_mentions... \n", + "3 {'hashtags': [], 'symbols': [], 'user_mentions... \n", + "4 {'hashtags': [], 'symbols': [], 'user_mentions... \n", + "\n", + " metadata \\\n", + "0 {'iso_language_code': 'en', 'result_type': 're... \n", + "1 {'iso_language_code': 'en', 'result_type': 're... \n", + "2 {'iso_language_code': 'en', 'result_type': 're... \n", + "3 {'iso_language_code': 'en', 'result_type': 're... \n", + "4 {'iso_language_code': 'en', 'result_type': 're... \n", + "\n", + " source in_reply_to_status_id \\\n", + "0 \n", + "RangeIndex: 100 entries, 0 to 99\n", + "Data columns (total 30 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 created_at 100 non-null object \n", + " 1 id 100 non-null int64 \n", + " 2 id_str 100 non-null object \n", + " 3 text 100 non-null object \n", + " 4 truncated 100 non-null bool \n", + " 5 entities 100 non-null object \n", + " 6 metadata 100 non-null object \n", + " 7 source 100 non-null object \n", + " 8 in_reply_to_status_id 7 non-null float64\n", + " 9 in_reply_to_status_id_str 7 non-null object \n", + " 10 in_reply_to_user_id 7 non-null float64\n", + " 11 in_reply_to_user_id_str 7 non-null object \n", + " 12 in_reply_to_screen_name 7 non-null object \n", + " 13 user 100 non-null object \n", + " 14 geo 0 non-null object \n", + " 15 coordinates 0 non-null object \n", + " 16 place 2 non-null object \n", + " 17 contributors 0 non-null object \n", + " 18 retweeted_status 83 non-null object \n", + " 19 is_quote_status 100 non-null bool \n", + " 20 retweet_count 100 non-null int64 \n", + " 21 favorite_count 100 non-null int64 \n", + " 22 favorited 100 non-null bool \n", + " 23 retweeted 100 non-null bool \n", + " 24 possibly_sensitive 17 non-null object \n", + " 25 lang 100 non-null object \n", + " 26 extended_entities 4 non-null object \n", + " 27 quoted_status_id 4 non-null float64\n", + " 28 quoted_status_id_str 4 non-null object \n", + " 29 quoted_status 2 non-null object \n", + "dtypes: bool(4), float64(3), int64(3), object(20)\n", + "memory usage: 20.8+ KB\n" + ] + } + ], + "source": [ + "df.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "import boto3\n", + "from io import StringIO\n", + "bucket_name = 'lab-03'\n", + "aws_s3_client = boto3.client('s3',\n", + " aws_access_key_id='AKIAQTX37VZOZMCNEMWF',\n", + " aws_secret_access_key= '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')" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "def upload_s3(df,i):\n", + " global aws_s3_client,bucket_name\n", + " csv_buf = StringIO()\n", + " df.to_csv(csv_buf,header=True,index=False)\n", + " csv_buf.seek(0)\n", + " aws_s3_client.put_object(Bucket=bucket_name,Body=csv_buf.getvalue(),Key=i)\n", + "\n", + "# # Check if the bucket exists\n", + "# response = aws_s3_client.list_buckets()\n", + "# buckets = [bucket['Name'] for bucket in response['Buckets']]\n", + "# if bucket_name not in buckets:\n", + "# print(f\"{bucket_name} bucket does not exist.\")\n", + "# else:\n", + "upload_s3(df, 'lab-03_Group7_04202023.csv')" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-spring-Group5_20220406220036.csv - 481896 bytes\n", + "Group4-Lab320220405195641.csv - 372403 bytes\n", + "Group_1_20220331201922.csv - 448076 bytes\n", + "Group_1_20220405195545.csv - 462455 bytes\n", + "Group_1_20220405195721.csv - 496262 bytes\n", + "Group_1_20220406191330.csv - 475513 bytes\n", + "Group_1_20230419213518.csv - 40611 bytes\n", + "Group_1_S20220327174953.csv - 458645 bytes\n", + "Group_3_20220406191456.csv - 487525 bytes\n", + "Group_Xiaolan_20210724.csv - 542269 bytes\n", + "Group_Y20220403101253.csv - 492660 bytes\n", + "Group_teli_20220402.csv - 499374 bytes\n", + "lab-03_Group7_04202023.csv - 488960 bytes\n", + "lab-3Group7.csv - 488960 bytes\n" + ] + } + ], + "source": [ + "# Get the objects in the S3 bucket\n", + "try:\n", + " response = aws_s3_client.list_objects_v2(Bucket=bucket_name)\n", + " if 'Contents' in response:\n", + " # Print the name and size of each object in the bucket\n", + " for obj in response['Contents']:\n", + " print(f\"{obj['Key']} - {obj['Size']} bytes\")\n", + " else:\n", + " print(f\"The bucket {bucket_name} is empty.\")\n", + "except Exception as e:\n", + " print(f\"Error listing objects in bucket {bucket_name}.\")\n", + " print(e)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/scripts/twitter-api-example/.ipynb_checkpoints/api-twitter-example-checkpoint.ipynb b/scripts/twitter-api-example/.ipynb_checkpoints/api-twitter-example-checkpoint.ipynb deleted file mode 100644 index dddd731..0000000 --- a/scripts/twitter-api-example/.ipynb_checkpoints/api-twitter-example-checkpoint.ipynb +++ /dev/null @@ -1,414 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting s3fs\n", - " Downloading s3fs-2023.4.0-py3-none-any.whl (28 kB)\n", - "Collecting aiohttp!=4.0.0a0,!=4.0.0a1\n", - " Downloading aiohttp-3.8.4-cp39-cp39-win_amd64.whl (323 kB)\n", - " ------------------------------------- 323.6/323.6 kB 20.9 MB/s eta 0:00:00\n", - "Collecting fsspec==2023.4.0\n", - " Downloading fsspec-2023.4.0-py3-none-any.whl (153 kB)\n", - " ---------------------------------------- 154.0/154.0 kB ? eta 0:00:00\n", - "Collecting aiobotocore~=2.5.0\n", - " Downloading aiobotocore-2.5.0-py3-none-any.whl (72 kB)\n", - " ---------------------------------------- 72.7/72.7 kB ? eta 0:00:00\n", - "Collecting aioitertools>=0.5.1\n", - " Downloading aioitertools-0.11.0-py3-none-any.whl (23 kB)\n", - "Collecting botocore<1.29.77,>=1.29.76\n", - " Downloading botocore-1.29.76-py3-none-any.whl (10.4 MB)\n", - " --------------------------------------- 10.4/10.4 MB 36.3 MB/s eta 0:00:00\n", - "Requirement already satisfied: wrapt>=1.10.10 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from aiobotocore~=2.5.0->s3fs) (1.14.1)\n", - "Collecting multidict<7.0,>=4.5\n", - " Downloading multidict-6.0.4-cp39-cp39-win_amd64.whl (28 kB)\n", - "Collecting async-timeout<5.0,>=4.0.0a3\n", - " Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n", - "Collecting yarl<2.0,>=1.0\n", - " Downloading yarl-1.8.2-cp39-cp39-win_amd64.whl (56 kB)\n", - " ---------------------------------------- 56.8/56.8 kB ? eta 0:00:00\n", - "Collecting aiosignal>=1.1.2\n", - " Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n", - "Collecting frozenlist>=1.1.1\n", - " Downloading frozenlist-1.3.3-cp39-cp39-win_amd64.whl (34 kB)\n", - "Requirement already satisfied: attrs>=17.3.0 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs) (21.4.0)\n", - "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs) (2.0.4)\n", - "Requirement already satisfied: typing_extensions>=4.0 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from aioitertools>=0.5.1->aiobotocore~=2.5.0->s3fs) (4.3.0)\n", - "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from botocore<1.29.77,>=1.29.76->aiobotocore~=2.5.0->s3fs) (0.10.0)\n", - "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from botocore<1.29.77,>=1.29.76->aiobotocore~=2.5.0->s3fs) (2.8.2)\n", - "Requirement already satisfied: urllib3<1.27,>=1.25.4 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from botocore<1.29.77,>=1.29.76->aiobotocore~=2.5.0->s3fs) (1.26.11)\n", - "Requirement already satisfied: idna>=2.0 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from yarl<2.0,>=1.0->aiohttp!=4.0.0a0,!=4.0.0a1->s3fs) (3.3)\n", - "Requirement already satisfied: six>=1.5 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.29.77,>=1.29.76->aiobotocore~=2.5.0->s3fs) (1.16.0)\n", - "Installing collected packages: multidict, fsspec, frozenlist, async-timeout, aioitertools, yarl, botocore, aiosignal, aiohttp, aiobotocore, s3fs\n", - " Attempting uninstall: fsspec\n", - " Found existing installation: fsspec 2022.7.1\n", - " Uninstalling fsspec-2022.7.1:\n", - " Successfully uninstalled fsspec-2022.7.1\n", - " Attempting uninstall: botocore\n", - " Found existing installation: botocore 1.27.28\n", - " Uninstalling botocore-1.27.28:\n", - " Successfully uninstalled botocore-1.27.28\n", - "Successfully installed aiobotocore-2.5.0 aiohttp-3.8.4 aioitertools-0.11.0 aiosignal-1.3.1 async-timeout-4.0.2 botocore-1.29.76 frozenlist-1.3.3 fsspec-2023.4.0 multidict-6.0.4 s3fs-2023.4.0 yarl-1.8.2\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", - "boto3 1.24.28 requires botocore<1.28.0,>=1.27.28, but you have botocore 1.29.76 which is incompatible.\n" - ] - }, - { - "data": { - "text/plain": [ - "200" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "\n", - "####################\n", - "#Author: brandon chiazza\n", - "#version: 1.0\n", - "#purpose: to call a twitter api and return results\n", - "#documentation: https://developer.twitter.com/en/docs\n", - "#####################\n", - "\n", - "import pandas as pd\n", - "import requests\n", - "import json\n", - "import base64\n", - "!pip install s3fs\n", - "import s3fs # documentation: https://s3fs.readthedocs.io/en/latest/\n", - "import time\n", - "import twitter_keys #this is a custom reference module to a package containing twitter keys\n", - "\n", - "%config IPCompleter.greedy=True\n", - "\n", - "\n", - "key_secret = '{}:{}'.format(twitter_keys.client_key, twitter_keys.client_secret).encode('ascii')\n", - "b64_encoded_key = base64.b64encode(key_secret)\n", - "b64_encoded_key = b64_encoded_key.decode('ascii')\n", - "\n", - "#identify base url and oauth token path\n", - "base_url = 'https://api.twitter.com/' #base url for authentication\n", - "auth_url = '{}oauth2/token'.format(base_url)\n", - "\n", - "#share header information -- encoding is ascii\n", - "auth_headers = {\n", - " 'Authorization': 'Basic {}'.format(b64_encoded_key),\n", - " 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'\n", - "}\n", - "\n", - "#pass clientcredentials\n", - "auth_data = {\n", - " 'grant_type': 'client_credentials'\n", - "}\n", - "\n", - "#send authentication using requests - POST request\n", - "auth_resp = requests.post(auth_url, headers=auth_headers, data=auth_data)\n", - "\n", - "#check response status. 200 = OK\n", - "auth_resp.status_code\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "dict_keys(['token_type', 'access_token'])\n" - ] - }, - { - "data": { - "text/plain": [ - "200" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "\n", - "#Keys in data response are token_type (bearer) and access_token (your access token)\n", - "print(auth_resp.json().keys())\n", - "\n", - "access_token = auth_resp.json()['access_token']\n", - "\n", - "\n", - "search_headers = {\n", - " 'Authorization': 'Bearer {}'.format(access_token) \n", - "}\n", - "\n", - "#enter search parameters for coronavirus example. This looks for \"covid-19\" in the 1000 most recent tweets\n", - "query_params = {\n", - " 'q': 'covid-19',\n", - " 'result_type': 'recent',\n", - " 'count': 100, #update here to get more/less than 1000 returns\n", - " 'lang': 'en' #filters by english language only\n", - "}\n", - "\n", - "\n", - "#identify search url path and save \n", - "search_url = '{}1.1/search/tweets.json'.format(base_url)\n", - "\n", - "\n", - "#run search using get request\n", - "search_resp = requests.get(search_url, headers=search_headers, params=query_params)\n", - "\n", - "#check status code of GET request\n", - "search_resp.status_code\n" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "RT @GURU11ji: 1\n", - "The integrity of Mr. Imran Khan is unquestionable, he is the man who gave the nation charitable cancer hospitals, universit…\n", - "\n" - ] - } - ], - "source": [ - "#print text from result to verify \n", - "twitter_data = search_resp.json()\n", - "\n", - "for x in twitter_data['statuses']:\n", - " print(x['text'] + '\\n')\n", - " break #prints after one iteration and stops, remove break to see all 1000" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
created_atidid_strtexttruncatedentitiesmetadatasourcein_reply_to_status_idin_reply_to_status_id_str...retweet_countfavorite_countfavoritedretweetedlangpossibly_sensitivequoted_status_idquoted_status_id_strquoted_statusextended_entities
0Sun Apr 16 02:52:58 +0000 202316474328705274839101647432870527483910RT @GURU11ji: 1\\nThe integrity of Mr. Imran Kh...False{'hashtags': [], 'symbols': [], 'user_mentions...{'iso_language_code': 'en', 'result_type': 're...<a href=\"http://twitter.com/download/android\" ...NaNNone...440FalseFalseenNaNNaNNaNNaNNaN
\n", - "

1 rows × 30 columns

\n", - "
" - ], - "text/plain": [ - " created_at id id_str \\\n", - "0 Sun Apr 16 02:52:58 +0000 2023 1647432870527483910 1647432870527483910 \n", - "\n", - " text truncated \\\n", - "0 RT @GURU11ji: 1\\nThe integrity of Mr. Imran Kh... False \n", - "\n", - " entities \\\n", - "0 {'hashtags': [], 'symbols': [], 'user_mentions... \n", - "\n", - " metadata \\\n", - "0 {'iso_language_code': 'en', 'result_type': 're... \n", - "\n", - " source in_reply_to_status_id \\\n", - "0 \u001b[1;34m\u001b[0m\n\u001b[0;32m 12\u001b[0m \u001b[0ms3\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0ms3fs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mS3FileSystem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0manon\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 13\u001b[0m \u001b[1;32mwith\u001b[0m \u001b[0ms3\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilenames3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wb'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mfile\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 14\u001b[1;33m \u001b[0mfile\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbyte_encoded_df\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m#writes byte-encoded file to s3 location\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 15\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 16\u001b[0m \u001b[1;31m#print success message\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\fsspec\\spec.py\u001b[0m in \u001b[0;36m__exit__\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m 1820\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1821\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m__exit__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1822\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\fsspec\\spec.py\u001b[0m in \u001b[0;36mclose\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 1787\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1788\u001b[0m \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforced\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1789\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflush\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mforce\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1790\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1791\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfs\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\fsspec\\spec.py\u001b[0m in \u001b[0;36mflush\u001b[1;34m(self, force)\u001b[0m\n\u001b[0;32m 1658\u001b[0m \u001b[1;32mraise\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1659\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1660\u001b[1;33m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_upload_chunk\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfinal\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mforce\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1661\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moffset\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mseek\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1662\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuffer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mio\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mBytesIO\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\s3fs\\core.py\u001b[0m in \u001b[0;36m_upload_chunk\u001b[1;34m(self, final)\u001b[0m\n\u001b[0;32m 2219\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 2220\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mautocommit\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfinal\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2221\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 2222\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mfinal\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 2223\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\s3fs\\core.py\u001b[0m in \u001b[0;36mcommit\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 2234\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mseek\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 2235\u001b[0m \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2236\u001b[1;33m write_result = self._call_s3(\n\u001b[0m\u001b[0;32m 2237\u001b[0m \u001b[1;34m\"put_object\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 2238\u001b[0m \u001b[0mKey\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\s3fs\\core.py\u001b[0m in \u001b[0;36m_call_s3\u001b[1;34m(self, method, *kwarglist, **kwargs)\u001b[0m\n\u001b[0;32m 2086\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 2087\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m_call_s3\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0mkwarglist\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2088\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcall_s3\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0ms3_additional_kwargs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0mkwarglist\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 2089\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 2090\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m_initiate_upload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\fsspec\\asyn.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m 113\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 114\u001b[0m \u001b[0mself\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mobj\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 115\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0msync\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloop\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 116\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 117\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\fsspec\\asyn.py\u001b[0m in \u001b[0;36msync\u001b[1;34m(loop, func, timeout, *args, **kwargs)\u001b[0m\n\u001b[0;32m 98\u001b[0m \u001b[1;32mraise\u001b[0m \u001b[0mFSTimeoutError\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mreturn_result\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 99\u001b[0m \u001b[1;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mreturn_result\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mBaseException\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 100\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mreturn_result\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 101\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 102\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mreturn_result\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\fsspec\\asyn.py\u001b[0m in \u001b[0;36m_runner\u001b[1;34m(event, coro, result, timeout)\u001b[0m\n\u001b[0;32m 53\u001b[0m \u001b[0mcoro\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0masyncio\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwait_for\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcoro\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 54\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 55\u001b[1;33m \u001b[0mresult\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mawait\u001b[0m \u001b[0mcoro\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 56\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mex\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 57\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mex\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\s3fs\\core.py\u001b[0m in \u001b[0;36m_call_s3\u001b[1;34m(self, method, *akwarglist, **kwargs)\u001b[0m\n\u001b[0;32m 346\u001b[0m \u001b[0mlogger\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"CALL: %s - %s - %s\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0makwarglist\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkw2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 347\u001b[0m \u001b[0madditional_kwargs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_s3_method_kwargs\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0makwarglist\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 348\u001b[1;33m return await _error_wrapper(\n\u001b[0m\u001b[0;32m 349\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0madditional_kwargs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretries\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mretries\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 350\u001b[0m )\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\s3fs\\core.py\u001b[0m in \u001b[0;36m_error_wrapper\u001b[1;34m(func, args, kwargs, retries)\u001b[0m\n\u001b[0;32m 138\u001b[0m \u001b[0merr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 139\u001b[0m \u001b[0merr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtranslate_boto_error\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 140\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 141\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 142\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\s3fs\\core.py\u001b[0m in \u001b[0;36m_error_wrapper\u001b[1;34m(func, args, kwargs, retries)\u001b[0m\n\u001b[0;32m 111\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mretries\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 112\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 113\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[1;32mawait\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 114\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mS3_RETRYABLE_ERRORS\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 115\u001b[0m \u001b[0merr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\aiobotocore\\client.py\u001b[0m in \u001b[0;36m_make_api_call\u001b[1;34m(self, operation_name, api_params)\u001b[0m\n\u001b[0;32m 352\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 353\u001b[0m \u001b[0mapply_request_checksum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrequest_dict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 354\u001b[1;33m http, parsed_response = await self._make_request(\n\u001b[0m\u001b[0;32m 355\u001b[0m \u001b[0moperation_model\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrequest_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrequest_context\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 356\u001b[0m )\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\aiobotocore\\client.py\u001b[0m in \u001b[0;36m_make_request\u001b[1;34m(self, operation_model, request_dict, request_context)\u001b[0m\n\u001b[0;32m 377\u001b[0m ):\n\u001b[0;32m 378\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 379\u001b[1;33m return await self._endpoint.make_request(\n\u001b[0m\u001b[0;32m 380\u001b[0m \u001b[0moperation_model\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrequest_dict\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 381\u001b[0m )\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\aiobotocore\\endpoint.py\u001b[0m in \u001b[0;36m_send_request\u001b[1;34m(self, request_dict, operation_model)\u001b[0m\n\u001b[0;32m 94\u001b[0m \u001b[0mcontext\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrequest_dict\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'context'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 95\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_update_retries_context\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcontext\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattempts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 96\u001b[1;33m \u001b[0mrequest\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mawait\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcreate_request\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrequest_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moperation_model\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 97\u001b[0m success_response, exception = await self._get_response(\n\u001b[0;32m 98\u001b[0m \u001b[0mrequest\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moperation_model\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcontext\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\aiobotocore\\endpoint.py\u001b[0m in \u001b[0;36mcreate_request\u001b[1;34m(self, params, operation_model)\u001b[0m\n\u001b[0;32m 82\u001b[0m \u001b[0mservice_id\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mservice_id\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mop_name\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moperation_model\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 83\u001b[0m )\n\u001b[1;32m---> 84\u001b[1;33m await self._event_emitter.emit(\n\u001b[0m\u001b[0;32m 85\u001b[0m \u001b[0mevent_name\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 86\u001b[0m \u001b[0mrequest\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mrequest\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\aiobotocore\\hooks.py\u001b[0m in \u001b[0;36m_emit\u001b[1;34m(self, event_name, kwargs, stop_on_response)\u001b[0m\n\u001b[0;32m 64\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 65\u001b[0m \u001b[1;31m# Await the handler if its a coroutine.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 66\u001b[1;33m \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mawait\u001b[0m \u001b[0mresolve_awaitable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhandler\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 67\u001b[0m \u001b[0mresponses\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhandler\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 68\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mstop_on_response\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mresponse\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\aiobotocore\\_helpers.py\u001b[0m in \u001b[0;36mresolve_awaitable\u001b[1;34m(obj)\u001b[0m\n\u001b[0;32m 13\u001b[0m \u001b[1;32masync\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mresolve_awaitable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 14\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0minspect\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misawaitable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 15\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[1;32mawait\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 16\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 17\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\aiobotocore\\signers.py\u001b[0m in \u001b[0;36mhandler\u001b[1;34m(self, operation_name, request, **kwargs)\u001b[0m\n\u001b[0;32m 22\u001b[0m \u001b[1;31m# this method is invoked to sign the request.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 23\u001b[0m \u001b[1;31m# Don't call this method directly.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 24\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[1;32mawait\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msign\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moperation_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 25\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 26\u001b[0m async def sign(\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\aiobotocore\\signers.py\u001b[0m in \u001b[0;36msign\u001b[1;34m(self, operation_name, request, region_name, signing_type, expires_in, signing_name)\u001b[0m\n\u001b[0;32m 80\u001b[0m \u001b[1;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 81\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 82\u001b[1;33m \u001b[0mauth\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd_auth\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 83\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 84\u001b[0m \u001b[1;32masync\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m_choose_signer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moperation_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msigning_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcontext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\botocore\\auth.py\u001b[0m in \u001b[0;36madd_auth\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 416\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0madd_auth\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 417\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcredentials\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 418\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mNoCredentialsError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 419\u001b[0m \u001b[0mdatetime_now\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdatetime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdatetime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mutcnow\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 420\u001b[0m \u001b[0mrequest\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontext\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'timestamp'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdatetime_now\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstrftime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mSIGV4_TIMESTAMP\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;31mNoCredentialsError\u001b[0m: Unable to locate credentials" - ] - } - ], - "source": [ - "# we can use pandas to put data directly into an s3 bucket\n", - "#prepare csv file name \n", - "filename = 's3lab-03'#specify location of s3:/{my-bucket}/\n", - "groupname= 'Group_7' #name of your group\n", - "datetime = time.strftime(\"%Y%m%d%H%M%S\") #timestamp\n", - "filenames3 = \"%s%s%s.csv\"%(filename,groupname,datetime) #name of the filepath and csv file\n", - "\n", - "\n", - "\n", - "#encoding must be adjusted to accommodate abnormal characters. Use s3fs to write to S3 bucket\n", - "byte_encoded_df = df.to_csv(None).encode() #encodes file as binary\n", - "s3 = s3fs.S3FileSystem(anon=False)\n", - "with s3.open(filenames3, 'wb') as file:\n", - " file.write(byte_encoded_df) #writes byte-encoded file to s3 location\n", - "\n", - "#print success message\n", - "print(\"Successfully uploaded file to location:\"+str(filenames3))\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/scripts/twitter-api-example/api-twitter-Group7.ipynb b/scripts/twitter-api-example/api-twitter-Group7.ipynb new file mode 100644 index 0000000..b7ccf86 --- /dev/null +++ b/scripts/twitter-api-example/api-twitter-Group7.ipynb @@ -0,0 +1,817 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "####################\n", + "#Author: brandon chiazza\n", + "#version: 1.0\n", + "#purpose: to call a twitter api and return results\n", + "#documentation: https://developer.twitter.com/en/docs\n", + "#####################\n", + "\n", + "import pandas as pd\n", + "import requests\n", + "import json\n", + "import base64\n", + "#!pip install s3fs\n", + "import s3fs # documentation: https://s3fs.readthedocs.io/en/latest/\n", + "import time\n", + "import twitter_keys #this is a custom reference module to a package containing twitter keys\n", + "\n", + "%config IPCompleter.greedy=True\n", + "\n", + "key_secret = '{}:{}'.format(twitter_keys.client_key, twitter_keys.client_secret).encode('ascii')\n", + "#key_secret = '{}:{}'.format('123', twitter_keys.client_secret).encode('ascii')\n", + "b64_encoded_key = base64.b64encode(key_secret)\n", + "b64_encoded_key = b64_encoded_key.decode('ascii')\n", + "\n", + "#identify base url and oauth token path\n", + "base_url = 'https://api.twitter.com/' #base url for authentication\n", + "auth_url = '{}oauth2/token'.format(base_url)\n", + "\n", + "#share header information -- encoding is ascii\n", + "auth_headers = {\n", + " 'Authorization': 'Basic {}'.format(b64_encoded_key),\n", + " 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'\n", + "}\n", + "\n", + "#pass clientcredentials\n", + "auth_data = {\n", + " 'grant_type': 'client_credentials'\n", + "}\n", + "\n", + "#send authentication using requests - POST request\n", + "auth_resp = requests.post(auth_url, headers=auth_headers, data=auth_data)\n", + "\n", + "#check response status. 200 = OK\n", + "auth_resp.status_code\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['token_type', 'access_token'])\n" + ] + }, + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "#Keys in data response are token_type (bearer) and access_token (your access token)\n", + "print(auth_resp.json().keys())\n", + "\n", + "access_token = auth_resp.json()['access_token']\n", + "\n", + "\n", + "search_headers = {\n", + " 'Authorization': 'Bearer {}'.format(access_token) \n", + "}\n", + "\n", + "#enter search parameters for coronavirus example. This looks for \"covid-19\" in the 1000 most recent tweets\n", + "query_params = {\n", + " 'q': 'covid-19',\n", + " 'result_type': 'recent',\n", + " 'count': 100, #update here to get more/less than 1000 returns\n", + " 'lang': 'en' #filters by english language only\n", + "}\n", + "\n", + "\n", + "#identify search url path and save \n", + "search_url = '{}1.1/search/tweets.json'.format(base_url)\n", + "\n", + "\n", + "#run search using get request\n", + "search_resp = requests.get(search_url, headers=search_headers, params=query_params)\n", + "\n", + "#check status code of GET request\n", + "search_resp.status_code\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "RT @soompi: #NCT_DOJAEJUNG Cancels This Week’s Promotions After #Jaehyun Experiences COVID-19 Symptoms\n", + "https://t.co/eVKT4JtqBX https://t.co…\n", + "\n", + "RT @KanekoaTheGreat: Former DNI John Ratcliffe says the CDC Director, the Secretary of State, and the Director of National Intelligence agr…\n", + "\n", + "RT @sanemiliomps: @sanemiliomps Prevention on Covid-19 https://t.co/9NdduKkvKo\n", + "\n", + "RT @laurieallee: From 2020:\n", + "\"1/3of patients with COVID-19 ― both active and cleared cases ― showed at least some measure of myocarditis.\" h…\n", + "\n", + "RT @MakisMD: Vista, CA - 40 year old Pastor Jay Foulk died in his sleep on April 13, 2023.\n", + "\n", + "He was negative for COVID-19, went to bed feeli…\n", + "\n", + "RT @sanemiliomps: @sanemiliomps COVID-19 SAFETY TIPS https://t.co/v7FG6Civ4P\n", + "\n", + "RT @stkirsch: Despite the fact that it will increase your chance of getting Covid, FDA authorizes another shot for older adults and people…\n", + "\n", + "RT @laralogan: These people are either literally insane or pure evil or both?\n", + "\n", + "It’s one thing for ordinary citizens to still be blinded by…\n", + "\n", + "RT @RogerMarshallMD: I sat down with @ABC and @TerryMoran to share the Origin of COVID-19 Report and why it is so important so we ensure th…\n", + "\n", + "RT @ShashiTharoor: On this issue ⁦@DrSJaishankar’s response surprises me. He knows that public representatives frequently⁩ convey to Govt a…\n", + "\n", + "RT @US_FDA: The monovalent Moderna and Pfizer-BioNTech COVID-19 vaccines are no longer authorized for use in the United States.\n", + "\n", + "RT @MakisMD: 43 year old UK Police Officer Andy Boardman collapsed while responding to a call and died suddenly on April 11, 2023.\n", + "\n", + "He leav…\n", + "\n", + "RT @TonyHinton2016: Doctor's death due to AstraZeneca Covid vaccine reaction - inquest - BBC News\n", + "\n", + "\"The death of a doctor after his Oxford-…\n", + "\n", + "RT @kevinnbass: In today's House Appropriations Committee hearing, CDC Director Rochelle Walensky lied directly to Congress, repeating her…\n", + "\n", + "RT @CourthouseNews: A Ninth Circuit panel on Wednesday confirmed the legality of President Joe Biden's Covid-19 vaccine mandate for federal…\n", + "\n", + "As federal aid winds down, long COVID patients feel left behind : Shots - Health News \n", + "\n", + "https://t.co/O8KRLfquff\n", + "\n", + "RT @COVIDSelect: 🔥🔥🔥\n", + " \n", + "“This has the potential to be one of the greatest medical disasters in world history, but also one of the greatest c…\n", + "\n", + "@Bucks @Marjon32 Guys Run and Dunk actually has 0 real awards 😭\n", + "2018-19 MVP: Only got it because he played on a sup… https://t.co/nfLi7njygj\n", + "\n", + "RT @US_FDA: The monovalent Moderna and Pfizer-BioNTech COVID-19 vaccines are no longer authorized for use in the United States.\n", + "\n", + "RT @RepThomasMassie: 2020: Candidate Biden called President Trump xenophobic for having a COVID-19 travel ban on China.\n", + "\n", + "But now that the C…\n", + "\n", + "RT @STVASTG: Right now in America, it is easier to get an AR-15 than a test kit for COVID-19. #shooter #gunviolence #guncontrolnow #gunviol…\n", + "\n", + "RT @albertan_angry: The real headline here should be that Canadians have zero right to medical privacy. Whatever you say to your doctor ca…\n", + "\n", + "RT @EdKrassen: As alleged COVID conspiracy theorist RFK Jr. today officially announced his candidacy for President, anti-vaxxers are once a…\n", + "\n", + "Biden's COVID vaccine rule for federal contractors was valid, US court rules https://t.co/OoQQOLlyyv… https://t.co/BTtYRuDzjS\n", + "\n", + "With the “tripledemic” (Covid/RSV/Flu) on the rise the Tasmanian Covid@homeplus program is keeping me and many othe… https://t.co/UkRGBse2gM\n", + "\n", + "RT @OccupyDemocrats: BREAKING: Marjorie Taylor Greene is publicly humiliated at a hearing on the origins of COVID-19 as Dr. Mark Lowenthal…\n", + "\n", + "Iloilo City Mayor Jerry Treñas on Wednesday issued an advisory reiterating national government policy on the wearin… https://t.co/3imKF093wO\n", + "\n", + "RT @MarkNeugebaue13: Exclusive: Australian Doctor Suspended for Speaking Out Against COVID-19 Vaccines Takes Medical Regulators to High Cou…\n", + "\n", + "RT @zerohedge: Watch: Former Director Of National Intelligence Admits That Fauci Lied About Gain Of Function Research https://t.co/CbEryBar…\n", + "\n", + "RT @MorningsMaria: Jamie Metzl urges Biden admin to ‘keep digging’ on origins of COVID-19 https://t.co/p6Id7PpeQ8\n", + "\n", + "RT @nytimes: Breaking News: The CDC recommended that adults 65 and older and those with weakened immune systems get another dose of the ref…\n", + "\n", + "RT @laralogan: These people are either literally insane or pure evil or both?\n", + "\n", + "It’s one thing for ordinary citizens to still be blinded by…\n", + "\n", + "RT @robster12065612: YOUR GOVERNMENT LIKES YOU THAT MUCH THEY PUSHED AN INJECTION ON YOU KNOWING THE COVID 19 WAS NEVER GOING TO KILL YOU B…\n", + "\n", + "RT @US_FDA: The monovalent Moderna and Pfizer-BioNTech COVID-19 vaccines are no longer authorized for use in the United States.\n", + "\n", + "RT @The_Trump_Train: WOW - The Moderna and Pfizer Covid 19 vaccines are NO LONGER AUTHORIZED for use in the United States. \n", + "\n", + "They must be f…\n", + "\n", + "RT @ZekeEmanuel: There is still no test, treatment, or cure for #LongCovid. There isn't even an accepted medical definition. We need to spe…\n", + "\n", + "RT @TonyHinton2016: Doctor's death due to AstraZeneca Covid vaccine reaction - inquest - BBC News\n", + "\n", + "\"The death of a doctor after his Oxford-…\n", + "\n", + "@elhoelho @therealrukshan So, @elhoelho did you know the @AstraZeneca COVID-19 “vaccine” product has been officiall… https://t.co/Fc1czrMa4E\n", + "\n", + "RT @kelcsimpkins: The CDC estimated in March that 6% of U.S. adults, or about 16 million, were experiencing #LongCovid, or ongoing health p…\n", + "\n", + "RT @Covid19Critical: \"The more shots you get, the greater your risk of getting COVID... so it has negative efficacy.\"\n", + "\n", + "Earlier this month,…\n", + "\n", + "RT @snopes: One popular social media post in the early days of the pandemic played on conspiracy theories by purporting to show a railroad…\n", + "\n", + "RT @Acyn: Moskowitz: I think the American people have a right to know why so much misinformation was spread about Covid 19 in this country…\n", + "\n", + "RT @kevinnbass: In today's House Appropriations Committee hearing, CDC Director Rochelle Walensky lied directly to Congress, repeating her…\n", + "\n", + "RT @DrGorfinkel: Think the pandemic’s over?\n", + "\n", + "COVID-19 is 🇨🇦’s 3rd leading cause of death. https://t.co/lbOEIujk41\n", + "\n", + "@NicoleMirante @RobertKennedyJr “I'm living in hell...\" Arrhythmia, discomfort, deep vein phlebectomy, a whole stri… https://t.co/13UiY2VYVm\n", + "\n", + "RT @kevinnbass: In today's House Appropriations Committee hearing, CDC Director Rochelle Walensky lied directly to Congress, repeating her…\n", + "\n", + "RT @CKellyUAP: 18,782 COVID DEATHS SINCE THE TGA BANNED IVERMECTIN.\n", + "\n", + "The TGA banned Ivermectin on the 10th September 2021, giving the most…\n", + "\n", + "@seanhannity Fauci deserve the highest reward medal and certificate for his courageous effort in combating COVID-19… https://t.co/jWGLnlcBp6\n", + "\n", + "RT @VPrasadMDMPH: I don't know if he plagiarized but I do know he is a terrible Covid-19 reporter. Arrogant and incurious.\n", + "\n", + "RT @zerohedge: Watch: Former Director Of National Intelligence Admits That Fauci Lied About Gain Of Function Research https://t.co/CbEryBar…\n", + "\n", + "RT @zerohedge: Watch: Former Director Of National Intelligence Admits That Fauci Lied About Gain Of Function Research https://t.co/CbEryBar…\n", + "\n", + "@covid19nz Collectively, this society doesn't have the workforce, the financial support, or the COURAGE to Unite Ag… https://t.co/5Llj1wZyAT\n", + "\n", + "RT @KanekoaTheGreat: 10/ Chris Krebs political bias and his affinity for domestic censorship:\n", + "\n", + "•Called Trump a national security threat\n", + "•Pl…\n", + "\n", + "RT @aiesecinghana: Her evolvement in the awareness of COVID 19 has been tremendous, she also donates to support child care https://t.co/5…\n", + "\n", + "RT @EdKrassen: As alleged COVID conspiracy theorist RFK Jr. today officially announced his candidacy for President, anti-vaxxers are once a…\n", + "\n", + "SciTechDaily: An Unlikely Hero: How the Common Cold Shields Children From COVID-19 https://t.co/ZP50eTr9PP\n", + "\n", + "One popular social media post in the early days of the pandemic played on conspiracy theories by purporting to show… https://t.co/6lbY7B1l2P\n", + "\n", + "@realLizUSA This is why Donald Trump lost the election. \n", + "\n", + "He will never be President again.\n", + "\n", + "Damning analysis of Tr… https://t.co/3X4bTWmQec\n", + "\n", + "COVID-19 virus, the global disaster and the public enemy of the world, has been sheltered from the lonely dream to… https://t.co/4VYmjnKJg4\n", + "\n", + "RT @DrLiMengYAN1: Takeaways on #COVID19 origin @COVIDSelect \n", + "1. Lab Leak ‘Only Explanation Credibly Supported by Intelligence, Science & Co…\n", + "\n", + "RT @kevinnbass: In today's House Appropriations Committee hearing, CDC Director Rochelle Walensky lied directly to Congress, repeating her…\n", + "\n", + "RT @zerohedge: Watch: Former Director Of National Intelligence Admits That Fauci Lied About Gain Of Function Research https://t.co/CbEryBar…\n", + "\n", + "RT @kevinnbass: In today's House Appropriations Committee hearing, CDC Director Rochelle Walensky lied directly to Congress, repeating her…\n", + "\n", + "RT @zerohedge: Watch: Former Director Of National Intelligence Admits That Fauci Lied About Gain Of Function Research https://t.co/CbEryBar…\n", + "\n", + "@JoshMatlow If only these great spaces were safe. #covidisairborne Want to LEAD? Show the city you're going to work… https://t.co/EoYg5cxMzy\n", + "\n", + "RT @rdumont99: For those who've used BC's \"COVID-19 Dashboard\", it wasn't updated this week and next week will be the last\n", + "\n", + "https://t.co/gd…\n", + "\n", + "Out of these, 421 children were diagnosed with COVID-19, and 3,298 children did not have COVID-19, but both groups… https://t.co/M8fESQCSNh\n", + "\n", + "RT @KelleyKga: In today's ACIP, CDC cited this KFF survey that found most people who had gotten a bivalent thought it was important to get…\n", + "\n", + "RT @captainuwu: \"Hello.\n", + "\n", + "We would like to tell you updates regarding Jaehyun's health.\n", + "\n", + "Jaehyun visited the hospital and had a COVID-19 PCR…\n", + "\n", + "RT @CNN: The Biden administration unveiled a $1.1 billion program aimed at providing Covid-19 vaccines and treatments to the uninsured at n…\n", + "\n", + "RT @kevinnbass: In today's House Appropriations Committee hearing, CDC Director Rochelle Walensky lied directly to Congress, repeating her…\n", + "\n", + "RT @KanekoaTheGreat: Former DNI John Ratcliffe says the CDC Director, the Secretary of State, and the Director of National Intelligence agr…\n", + "\n", + "RT @nytimes: Breaking News: The CDC recommended that adults 65 and older and those with weakened immune systems get another dose of the ref…\n", + "\n", + "U.S. Authorizes a New Round of Covid Boosters https://t.co/hiBujni2Hs\n", + "\n", + "RT @c_s_wallace: ‘Wearing masks was common in Japan even before the COVID-19 pandemic, and some people regularly use masks during hay fever…\n", + "\n", + "RT @nytimes: Breaking News: The CDC recommended that adults 65 and older and those with weakened immune systems get another dose of the ref…\n", + "\n", + "RT @NFSC999: Zhang Yutong, a 5-year-old boy from China's Henan province, was diagnosed with acute leukemia after an examination for weaknes…\n", + "\n", + "RT @mtaibbi: 1.TWITTER FILES #19\n", + "The Great Covid-19 Lie Machine\n", + "Stanford, the Virality Project, and the Censorship of “True Stories” https:…\n", + "\n", + "RT @MakisMD: 43 year old UK Police Officer Andy Boardman collapsed while responding to a call and died suddenly on April 11, 2023.\n", + "\n", + "He leav…\n", + "\n", + "RT @B52Malmet: CDC greenlights spring COVID booster for some. Do you need it? I’m getting mine Monday. That’ll be six shots for me all toge…\n", + "\n", + "RT @MakisMD: Vista, CA - 40 year old Pastor Jay Foulk died in his sleep on April 13, 2023.\n", + "\n", + "He was negative for COVID-19, went to bed feeli…\n", + "\n", + "RT @JoshDehaas: In April 2023, doctors in Alberta have again denied Anette Lewis access to the transplant list because she won’t accept COV…\n", + "\n", + "RT @Lidsville: \"Hospitalizations for COVID-19 among Canada’s children & youth surged by more than 600% in the 2nd year of the pandemic\"\n", + "\n", + "Re…\n", + "\n", + "RT @LionAdvocacy: Today is April 18th, 2023.\n", + "\n", + "Today, the FDA announced the monovalent Pfizer & Moderna jabs are no longer authorized.\n", + "\n", + "Toda…\n", + "\n", + "RT @EdKrassen: As alleged COVID conspiracy theorist RFK Jr. today officially announced his candidacy for President, anti-vaxxers are once a…\n", + "\n", + "RT @stkirsch: Despite the fact that it will increase your chance of getting Covid, FDA authorizes another shot for older adults and people…\n", + "\n", + "RT @B52Malmet: CDC greenlights spring COVID booster for some. Do you need it? I’m getting mine Monday. That’ll be six shots for me all toge…\n", + "\n", + "#LongCovid NOT psychosomatic:\n", + "\n", + "”Our results should not be misinterpreted as supporting a hypothesis that post–COVID… https://t.co/5eeRK5vwhX\n", + "\n", + "RT @CKellyUAP: 18,782 COVID DEATHS SINCE THE TGA BANNED IVERMECTIN.\n", + "\n", + "The TGA banned Ivermectin on the 10th September 2021, giving the most…\n", + "\n", + "RT @TonyHinton2016: Doctor's death due to AstraZeneca Covid vaccine reaction - inquest - BBC News\n", + "\n", + "\"The death of a doctor after his Oxford-…\n", + "\n", + "RT @WebMD: New research is shedding light on how an infection with COVID-19 may reactivate, or even cause, psoriasis.\n", + "https://t.co/pZ6nNDi1…\n", + "\n", + "RT @HopeRising19: How much #nz $$ were spent on the Unite Against Covid 19 and the Vaccine promotion campaign?\n", + "\n", + "Just marketing \n", + "\n", + "From April…\n", + "\n", + "RT @DailyClout: Children's Health Defense: The U.S. government approved its first three payments to people injured by COVID-19 vaccines — a…\n", + "\n", + "RT @Teagan1776: Did you know?? Todays News that went under/unreported- The monovalent Moderna & Pfizer BioNTech Covid-19 “vaccines” are no…\n", + "\n", + "RT @Storiesofinjury: A 56 year old math teacher had an Intracranial Hemorrhage after the first dose of Pfizer in March of 2023. She unfort…\n", + "\n", + "RT @De_Spectator_: Ex Director of National Intelligence John Ratcliffe says there’s overwhelming evidence that COVID-19 was manufactured in…\n", + "\n", + "RT @McMasterU: COVID-19 is the third-leading cause of death in Canada, but it’s older people who are dying. That we accept this and carry o…\n", + "\n", + "RT @michaelpsenger: “A physician who fled communist Yugoslavia saw the COVID-19 pandemic ‘through two eyes’…lockdowns were imposed…‘That wa…\n", + "\n", + "RT @Roman_Baber: \"The death of a doctor after his Oxford-AstraZeneca Covid-19 jab was due to unintended complications of the vaccine, an in…\n", + "\n", + "RT @MackayIM: If 1% of COVID-19 cases result in death, does that mean you have a 1% chance of dying if you catch it? A mathematician explai…\n", + "\n" + ] + } + ], + "source": [ + "#print text from result to verify \n", + "twitter_data = search_resp.json()\n", + "\n", + "for x in twitter_data['statuses']:\n", + " print(x['text'] + '\\n')\n", + " #break #prints after one iteration and stops, remove break to see all 1000" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
created_atidid_strtexttruncatedentitiesmetadatasourcein_reply_to_status_idin_reply_to_status_id_str...retweet_countfavorite_countfavoritedretweetedpossibly_sensitivelangextended_entitiesquoted_status_idquoted_status_id_strquoted_status
0Thu Apr 20 00:56:50 +0000 202316488531929799557151648853192979955715RT @soompi: #NCT_DOJAEJUNG Cancels This Week’s...False{'hashtags': [{'text': 'NCT_DOJAEJUNG', 'indic...{'iso_language_code': 'en', 'result_type': 're...<a href=\"http://twitter.com/download/android\" ...NaNNone...2030FalseFalseFalseenNaNNaNNaNNaN
1Thu Apr 20 00:56:49 +0000 202316488531924894597121648853192489459712RT @KanekoaTheGreat: Former DNI John Ratcliffe...False{'hashtags': [], 'symbols': [], 'user_mentions...{'iso_language_code': 'en', 'result_type': 're...<a href=\"http://twitter.com/download/android\" ...NaNNone...170450FalseFalseNaNenNaNNaNNaNNaN
2Thu Apr 20 00:56:47 +0000 202316488531813242183681648853181324218368RT @sanemiliomps: @sanemiliomps Prevention on ...False{'hashtags': [], 'symbols': [], 'user_mentions...{'iso_language_code': 'en', 'result_type': 're...<a href=\"http://twitter.com/download/android\" ...NaNNone...20FalseFalseFalseen{'media': [{'id': 1648176605196603394, 'id_str...NaNNaNNaN
3Thu Apr 20 00:56:47 +0000 202316488531807789547541648853180778954754RT @laurieallee: From 2020:\\n\"1/3of patients w...False{'hashtags': [], 'symbols': [], 'user_mentions...{'iso_language_code': 'en', 'result_type': 're...<a href=\"http://twitter.com/download/android\" ...NaNNone...3000FalseFalseNaNenNaNNaNNaNNaN
4Thu Apr 20 00:56:46 +0000 202316488531780189306911648853178018930691RT @MakisMD: Vista, CA - 40 year old Pastor Ja...False{'hashtags': [], 'symbols': [], 'user_mentions...{'iso_language_code': 'en', 'result_type': 're...<a href=\"https://mobile.twitter.com\" rel=\"nofo...NaNNone...6970FalseFalseNaNenNaNNaNNaNNaN
\n", + "

5 rows × 30 columns

\n", + "
" + ], + "text/plain": [ + " created_at id id_str \\\n", + "0 Thu Apr 20 00:56:50 +0000 2023 1648853192979955715 1648853192979955715 \n", + "1 Thu Apr 20 00:56:49 +0000 2023 1648853192489459712 1648853192489459712 \n", + "2 Thu Apr 20 00:56:47 +0000 2023 1648853181324218368 1648853181324218368 \n", + "3 Thu Apr 20 00:56:47 +0000 2023 1648853180778954754 1648853180778954754 \n", + "4 Thu Apr 20 00:56:46 +0000 2023 1648853178018930691 1648853178018930691 \n", + "\n", + " text truncated \\\n", + "0 RT @soompi: #NCT_DOJAEJUNG Cancels This Week’s... False \n", + "1 RT @KanekoaTheGreat: Former DNI John Ratcliffe... False \n", + "2 RT @sanemiliomps: @sanemiliomps Prevention on ... False \n", + "3 RT @laurieallee: From 2020:\\n\"1/3of patients w... False \n", + "4 RT @MakisMD: Vista, CA - 40 year old Pastor Ja... False \n", + "\n", + " entities \\\n", + "0 {'hashtags': [{'text': 'NCT_DOJAEJUNG', 'indic... \n", + "1 {'hashtags': [], 'symbols': [], 'user_mentions... \n", + "2 {'hashtags': [], 'symbols': [], 'user_mentions... \n", + "3 {'hashtags': [], 'symbols': [], 'user_mentions... \n", + "4 {'hashtags': [], 'symbols': [], 'user_mentions... \n", + "\n", + " metadata \\\n", + "0 {'iso_language_code': 'en', 'result_type': 're... \n", + "1 {'iso_language_code': 'en', 'result_type': 're... \n", + "2 {'iso_language_code': 'en', 'result_type': 're... \n", + "3 {'iso_language_code': 'en', 'result_type': 're... \n", + "4 {'iso_language_code': 'en', 'result_type': 're... \n", + "\n", + " source in_reply_to_status_id \\\n", + "0 \n", + "RangeIndex: 100 entries, 0 to 99\n", + "Data columns (total 30 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 created_at 100 non-null object \n", + " 1 id 100 non-null int64 \n", + " 2 id_str 100 non-null object \n", + " 3 text 100 non-null object \n", + " 4 truncated 100 non-null bool \n", + " 5 entities 100 non-null object \n", + " 6 metadata 100 non-null object \n", + " 7 source 100 non-null object \n", + " 8 in_reply_to_status_id 7 non-null float64\n", + " 9 in_reply_to_status_id_str 7 non-null object \n", + " 10 in_reply_to_user_id 7 non-null float64\n", + " 11 in_reply_to_user_id_str 7 non-null object \n", + " 12 in_reply_to_screen_name 7 non-null object \n", + " 13 user 100 non-null object \n", + " 14 geo 0 non-null object \n", + " 15 coordinates 0 non-null object \n", + " 16 place 2 non-null object \n", + " 17 contributors 0 non-null object \n", + " 18 retweeted_status 83 non-null object \n", + " 19 is_quote_status 100 non-null bool \n", + " 20 retweet_count 100 non-null int64 \n", + " 21 favorite_count 100 non-null int64 \n", + " 22 favorited 100 non-null bool \n", + " 23 retweeted 100 non-null bool \n", + " 24 possibly_sensitive 17 non-null object \n", + " 25 lang 100 non-null object \n", + " 26 extended_entities 4 non-null object \n", + " 27 quoted_status_id 4 non-null float64\n", + " 28 quoted_status_id_str 4 non-null object \n", + " 29 quoted_status 2 non-null object \n", + "dtypes: bool(4), float64(3), int64(3), object(20)\n", + "memory usage: 20.8+ KB\n" + ] + } + ], + "source": [ + "df.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "import boto3\n", + "from io import StringIO\n", + "bucket_name = 'lab-03'\n", + "aws_s3_client = boto3.client('s3',\n", + " aws_access_key_id='AKIAQTX37VZOZMCNEMWF',\n", + " aws_secret_access_key= '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')" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "def upload_s3(df,i):\n", + " global aws_s3_client,bucket_name\n", + " csv_buf = StringIO()\n", + " df.to_csv(csv_buf,header=True,index=False)\n", + " csv_buf.seek(0)\n", + " aws_s3_client.put_object(Bucket=bucket_name,Body=csv_buf.getvalue(),Key=i)\n", + "\n", + "# # Check if the bucket exists\n", + "# response = aws_s3_client.list_buckets()\n", + "# buckets = [bucket['Name'] for bucket in response['Buckets']]\n", + "# if bucket_name not in buckets:\n", + "# print(f\"{bucket_name} bucket does not exist.\")\n", + "# else:\n", + "upload_s3(df, 'lab-03_Group7_04202023.csv')" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-spring-Group5_20220406220036.csv - 481896 bytes\n", + "Group4-Lab320220405195641.csv - 372403 bytes\n", + "Group_1_20220331201922.csv - 448076 bytes\n", + "Group_1_20220405195545.csv - 462455 bytes\n", + "Group_1_20220405195721.csv - 496262 bytes\n", + "Group_1_20220406191330.csv - 475513 bytes\n", + "Group_1_20230419213518.csv - 40611 bytes\n", + "Group_1_S20220327174953.csv - 458645 bytes\n", + "Group_3_20220406191456.csv - 487525 bytes\n", + "Group_Xiaolan_20210724.csv - 542269 bytes\n", + "Group_Y20220403101253.csv - 492660 bytes\n", + "Group_teli_20220402.csv - 499374 bytes\n", + "lab-03_Group7_04202023.csv - 488960 bytes\n", + "lab-3Group7.csv - 488960 bytes\n" + ] + } + ], + "source": [ + "# Get the objects in the S3 bucket\n", + "try:\n", + " response = aws_s3_client.list_objects_v2(Bucket=bucket_name)\n", + " if 'Contents' in response:\n", + " # Print the name and size of each object in the bucket\n", + " for obj in response['Contents']:\n", + " print(f\"{obj['Key']} - {obj['Size']} bytes\")\n", + " else:\n", + " print(f\"The bucket {bucket_name} is empty.\")\n", + "except Exception as e:\n", + " print(f\"Error listing objects in bucket {bucket_name}.\")\n", + " print(e)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/scripts/twitter-api-example/api-twitter-example.ipynb b/scripts/twitter-api-example/api-twitter-example.ipynb deleted file mode 100644 index 07a885e..0000000 --- a/scripts/twitter-api-example/api-twitter-example.ipynb +++ /dev/null @@ -1,396 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting s3fs\n", - " Downloading s3fs-2023.4.0-py3-none-any.whl (28 kB)\n", - "Collecting aiohttp!=4.0.0a0,!=4.0.0a1\n", - " Downloading aiohttp-3.8.4-cp39-cp39-win_amd64.whl (323 kB)\n", - " ------------------------------------- 323.6/323.6 kB 20.9 MB/s eta 0:00:00\n", - "Collecting fsspec==2023.4.0\n", - " Downloading fsspec-2023.4.0-py3-none-any.whl (153 kB)\n", - " ---------------------------------------- 154.0/154.0 kB ? eta 0:00:00\n", - "Collecting aiobotocore~=2.5.0\n", - " Downloading aiobotocore-2.5.0-py3-none-any.whl (72 kB)\n", - " ---------------------------------------- 72.7/72.7 kB ? eta 0:00:00\n", - "Collecting aioitertools>=0.5.1\n", - " Downloading aioitertools-0.11.0-py3-none-any.whl (23 kB)\n", - "Collecting botocore<1.29.77,>=1.29.76\n", - " Downloading botocore-1.29.76-py3-none-any.whl (10.4 MB)\n", - " --------------------------------------- 10.4/10.4 MB 36.3 MB/s eta 0:00:00\n", - "Requirement already satisfied: wrapt>=1.10.10 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from aiobotocore~=2.5.0->s3fs) (1.14.1)\n", - "Collecting multidict<7.0,>=4.5\n", - " Downloading multidict-6.0.4-cp39-cp39-win_amd64.whl (28 kB)\n", - "Collecting async-timeout<5.0,>=4.0.0a3\n", - " Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n", - "Collecting yarl<2.0,>=1.0\n", - " Downloading yarl-1.8.2-cp39-cp39-win_amd64.whl (56 kB)\n", - " ---------------------------------------- 56.8/56.8 kB ? eta 0:00:00\n", - "Collecting aiosignal>=1.1.2\n", - " Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n", - "Collecting frozenlist>=1.1.1\n", - " Downloading frozenlist-1.3.3-cp39-cp39-win_amd64.whl (34 kB)\n", - "Requirement already satisfied: attrs>=17.3.0 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs) (21.4.0)\n", - "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs) (2.0.4)\n", - "Requirement already satisfied: typing_extensions>=4.0 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from aioitertools>=0.5.1->aiobotocore~=2.5.0->s3fs) (4.3.0)\n", - "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from botocore<1.29.77,>=1.29.76->aiobotocore~=2.5.0->s3fs) (0.10.0)\n", - "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from botocore<1.29.77,>=1.29.76->aiobotocore~=2.5.0->s3fs) (2.8.2)\n", - "Requirement already satisfied: urllib3<1.27,>=1.25.4 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from botocore<1.29.77,>=1.29.76->aiobotocore~=2.5.0->s3fs) (1.26.11)\n", - "Requirement already satisfied: idna>=2.0 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from yarl<2.0,>=1.0->aiohttp!=4.0.0a0,!=4.0.0a1->s3fs) (3.3)\n", - "Requirement already satisfied: six>=1.5 in c:\\users\\lakshmikar reddy\\anaconda3\\lib\\site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.29.77,>=1.29.76->aiobotocore~=2.5.0->s3fs) (1.16.0)\n", - "Installing collected packages: multidict, fsspec, frozenlist, async-timeout, aioitertools, yarl, botocore, aiosignal, aiohttp, aiobotocore, s3fs\n", - " Attempting uninstall: fsspec\n", - " Found existing installation: fsspec 2022.7.1\n", - " Uninstalling fsspec-2022.7.1:\n", - " Successfully uninstalled fsspec-2022.7.1\n", - " Attempting uninstall: botocore\n", - " Found existing installation: botocore 1.27.28\n", - " Uninstalling botocore-1.27.28:\n", - " Successfully uninstalled botocore-1.27.28\n", - "Successfully installed aiobotocore-2.5.0 aiohttp-3.8.4 aioitertools-0.11.0 aiosignal-1.3.1 async-timeout-4.0.2 botocore-1.29.76 frozenlist-1.3.3 fsspec-2023.4.0 multidict-6.0.4 s3fs-2023.4.0 yarl-1.8.2\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", - "boto3 1.24.28 requires botocore<1.28.0,>=1.27.28, but you have botocore 1.29.76 which is incompatible.\n" - ] - }, - { - "data": { - "text/plain": [ - "200" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "\n", - "####################\n", - "#Author: brandon chiazza\n", - "#version: 1.0\n", - "#purpose: to call a twitter api and return results\n", - "#documentation: https://developer.twitter.com/en/docs\n", - "#####################\n", - "\n", - "import pandas as pd\n", - "import requests\n", - "import json\n", - "import base64\n", - "!pip install s3fs\n", - "import s3fs # documentation: https://s3fs.readthedocs.io/en/latest/\n", - "import time\n", - "import twitter_keys #this is a custom reference module to a package containing twitter keys\n", - "\n", - "%config IPCompleter.greedy=True\n", - "\n", - "\n", - "key_secret = '{}:{}'.format(twitter_keys.client_key, twitter_keys.client_secret).encode('ascii')\n", - "b64_encoded_key = base64.b64encode(key_secret)\n", - "b64_encoded_key = b64_encoded_key.decode('ascii')\n", - "\n", - "#identify base url and oauth token path\n", - "base_url = 'https://api.twitter.com/' #base url for authentication\n", - "auth_url = '{}oauth2/token'.format(base_url)\n", - "\n", - "#share header information -- encoding is ascii\n", - "auth_headers = {\n", - " 'Authorization': 'Basic {}'.format(b64_encoded_key),\n", - " 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'\n", - "}\n", - "\n", - "#pass clientcredentials\n", - "auth_data = {\n", - " 'grant_type': 'client_credentials'\n", - "}\n", - "\n", - "#send authentication using requests - POST request\n", - "auth_resp = requests.post(auth_url, headers=auth_headers, data=auth_data)\n", - "\n", - "#check response status. 200 = OK\n", - "auth_resp.status_code\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "dict_keys(['token_type', 'access_token'])\n" - ] - }, - { - "data": { - "text/plain": [ - "200" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "\n", - "#Keys in data response are token_type (bearer) and access_token (your access token)\n", - "print(auth_resp.json().keys())\n", - "\n", - "access_token = auth_resp.json()['access_token']\n", - "\n", - "\n", - "search_headers = {\n", - " 'Authorization': 'Bearer {}'.format(access_token) \n", - "}\n", - "\n", - "#enter search parameters for coronavirus example. This looks for \"covid-19\" in the 1000 most recent tweets\n", - "query_params = {\n", - " 'q': 'covid-19',\n", - " 'result_type': 'recent',\n", - " 'count': 100, #update here to get more/less than 1000 returns\n", - " 'lang': 'en' #filters by english language only\n", - "}\n", - "\n", - "\n", - "#identify search url path and save \n", - "search_url = '{}1.1/search/tweets.json'.format(base_url)\n", - "\n", - "\n", - "#run search using get request\n", - "search_resp = requests.get(search_url, headers=search_headers, params=query_params)\n", - "\n", - "#check status code of GET request\n", - "search_resp.status_code\n" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "RT @GURU11ji: 1\n", - "The integrity of Mr. Imran Khan is unquestionable, he is the man who gave the nation charitable cancer hospitals, universit…\n", - "\n" - ] - } - ], - "source": [ - "#print text from result to verify \n", - "twitter_data = search_resp.json()\n", - "\n", - "for x in twitter_data['statuses']:\n", - " print(x['text'] + '\\n')\n", - " break #prints after one iteration and stops, remove break to see all 1000" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
created_atidid_strtexttruncatedentitiesmetadatasourcein_reply_to_status_idin_reply_to_status_id_str...retweet_countfavorite_countfavoritedretweetedlangpossibly_sensitivequoted_status_idquoted_status_id_strquoted_statusextended_entities
0Sun Apr 16 02:52:58 +0000 202316474328705274839101647432870527483910RT @GURU11ji: 1\\nThe integrity of Mr. Imran Kh...False{'hashtags': [], 'symbols': [], 'user_mentions...{'iso_language_code': 'en', 'result_type': 're...<a href=\"http://twitter.com/download/android\" ...NaNNone...440FalseFalseenNaNNaNNaNNaNNaN
\n", - "

1 rows × 30 columns

\n", - "
" - ], - "text/plain": [ - " created_at id id_str \\\n", - "0 Sun Apr 16 02:52:58 +0000 2023 1647432870527483910 1647432870527483910 \n", - "\n", - " text truncated \\\n", - "0 RT @GURU11ji: 1\\nThe integrity of Mr. Imran Kh... False \n", - "\n", - " entities \\\n", - "0 {'hashtags': [], 'symbols': [], 'user_mentions... \n", - "\n", - " metadata \\\n", - "0 {'iso_language_code': 'en', 'result_type': 're... \n", - "\n", - " source in_reply_to_status_id \\\n", - "0 \u001b[1;34m\u001b[0m\n\u001b[0;32m 11\u001b[0m \u001b[0mbyte_encoded_df\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m#encodes file as binary\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 12\u001b[0m \u001b[0ms3\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0ms3fs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mS3FileSystem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0manon\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 13\u001b[1;33m \u001b[1;32mwith\u001b[0m \u001b[0ms3\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilenames3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wb'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mfile\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 14\u001b[0m \u001b[0mfile\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbyte_encoded_df\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m#writes byte-encoded file to s3 location\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 15\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\fsspec\\spec.py\u001b[0m in \u001b[0;36mopen\u001b[1;34m(self, path, mode, block_size, cache_options, compression, **kwargs)\u001b[0m\n\u001b[0;32m 1152\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1153\u001b[0m \u001b[0mac\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"autocommit\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_intrans\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1154\u001b[1;33m f = self._open(\n\u001b[0m\u001b[0;32m 1155\u001b[0m \u001b[0mpath\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1156\u001b[0m \u001b[0mmode\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmode\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\s3fs\\core.py\u001b[0m in \u001b[0;36m_open\u001b[1;34m(self, path, mode, block_size, acl, version_id, fill_cache, cache_type, autocommit, requester_pays, cache_options, **kwargs)\u001b[0m\n\u001b[0;32m 657\u001b[0m \u001b[0mcache_type\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdefault_cache_type\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 658\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 659\u001b[1;33m return S3File(\n\u001b[0m\u001b[0;32m 660\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 661\u001b[0m \u001b[0mpath\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\s3fs\\core.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, s3, path, mode, block_size, acl, version_id, fill_cache, s3_additional_kwargs, autocommit, cache_type, requester_pays, cache_options)\u001b[0m\n\u001b[0;32m 2008\u001b[0m \u001b[0mbucket\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpath_version_id\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0ms3\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit_path\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 2009\u001b[0m \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2010\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Attempt to open non key-like path: %s\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mpath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 2011\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbucket\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbucket\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 2012\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkey\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;31mValueError\u001b[0m: Attempt to open non key-like path: s3lab-03Group_720230415234520.csv" - ] - } - ], - "source": [ - "# we can use pandas to put data directly into an s3 bucket\n", - "#prepare csv file name \n", - "filename = 's3lab-03'#specify location of s3:/{my-bucket}/\n", - "groupname= 'Group_7' #name of your group\n", - "datetime = time.strftime(\"%Y%m%d%H%M%S\") #timestamp\n", - "filenames3 = \"%s%s%s.csv\"%(filename,groupname,datetime) #name of the filepath and csv file\n", - "\n", - "\n", - "\n", - "#encoding must be adjusted to accommodate abnormal characters. Use s3fs to write to S3 bucket\n", - "byte_encoded_df = df.to_csv(None).encode() #encodes file as binary\n", - "s3 = s3fs.S3FileSystem(anon=False)\n", - "with s3.open(filenames3, 'wb') as file:\n", - " file.write(byte_encoded_df) #writes byte-encoded file to s3 location\n", - "\n", - "#print success message\n", - "print(\"Successfully uploaded file to location:\"+str(filenames3))\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -}