|
| 1 | +{ |
| 2 | + "cells": [ |
| 3 | + { |
| 4 | + "cell_type": "code", |
| 5 | + "execution_count": null, |
| 6 | + "metadata": {}, |
| 7 | + "outputs": [], |
| 8 | + "source": [ |
| 9 | + "from tweepy.streaming import StreamListener\n", |
| 10 | + "from tweepy import OAuthHandler\n", |
| 11 | + "from tweepy import Stream\n", |
| 12 | + "from kafka import SimpleProducer, KafkaClient\n", |
| 13 | + "import time\n", |
| 14 | + "\n", |
| 15 | + "access_token = \"748200460067045376-zYxRRyxiPIywcw2IV50IQiIxzQVN5FZ\"\n", |
| 16 | + "access_token_secret = \"c6dRkeRbgPqtbWAOTz0OfOMpBvhZS6KqWFjtEqHBEv7me\"\n", |
| 17 | + "consumer_key = \"ukUrCrJdd6MQQd0HQBzCDwcLq\"\n", |
| 18 | + "consumer_secret = \"VCf2wU1MhedUFnQeCwffzstdVkF7rbURzoNNDAdPPvbWfDtggP\"\n", |
| 19 | + "kafka_endpoint = \"ip-20-0-32-4.ap-south-1.compute.internal:9092\"\n", |
| 20 | + "kafka_topic = \"rk_hadoop\"\n", |
| 21 | + "twitter_hash_tag = \"RamNavami\"\n", |
| 22 | + "time_limit = 10\n", |
| 23 | + "\n", |
| 24 | + "class StdOutListener(StreamListener):\n", |
| 25 | + "\tdef __init__(self, time_limit=time_limit):\n", |
| 26 | + "\t\tself.start_time = time.time()\n", |
| 27 | + "\t\tself.limit = time_limit\n", |
| 28 | + "\t\tsuper(StdOutListener, self).__init__()\n", |
| 29 | + "\tdef on_data(self, data):\n", |
| 30 | + "\t\tif (time.time() - self.start_time) < self.limit:\n", |
| 31 | + " #msg = json.loads(data)\n", |
| 32 | + "\t\t\tproducer.send_messages(kafka_topic, data.encode('utf-8'))\n", |
| 33 | + "\t\t\tprint (data)\n", |
| 34 | + "\t\t\treturn True\n", |
| 35 | + "\t\texit(0)\n", |
| 36 | + "\tdef on_error(self, status):\n", |
| 37 | + "\t\tprint (status)\n", |
| 38 | + "\n", |
| 39 | + "kafka = KafkaClient(kafka_endpoint)\n", |
| 40 | + "producer = SimpleProducer(kafka)\n", |
| 41 | + "l = StdOutListener()\n", |
| 42 | + "auth = OAuthHandler(consumer_key, consumer_secret)\n", |
| 43 | + "auth.set_access_token(access_token, access_token_secret)\n", |
| 44 | + "stream = Stream(auth, l)\n", |
| 45 | + "stream.filter(track=twitter_hash_tag)\n" |
| 46 | + ] |
| 47 | + }, |
| 48 | + { |
| 49 | + "cell_type": "code", |
| 50 | + "execution_count": null, |
| 51 | + "metadata": { |
| 52 | + "collapsed": true |
| 53 | + }, |
| 54 | + "outputs": [], |
| 55 | + "source": [] |
| 56 | + }, |
| 57 | + { |
| 58 | + "cell_type": "code", |
| 59 | + "execution_count": null, |
| 60 | + "metadata": { |
| 61 | + "collapsed": true |
| 62 | + }, |
| 63 | + "outputs": [], |
| 64 | + "source": [] |
| 65 | + } |
| 66 | + ], |
| 67 | + "metadata": { |
| 68 | + "kernelspec": { |
| 69 | + "display_name": "Python 3", |
| 70 | + "language": "python", |
| 71 | + "name": "python3" |
| 72 | + }, |
| 73 | + "language_info": { |
| 74 | + "codemirror_mode": { |
| 75 | + "name": "ipython", |
| 76 | + "version": 3 |
| 77 | + }, |
| 78 | + "file_extension": ".py", |
| 79 | + "mimetype": "text/x-python", |
| 80 | + "name": "python", |
| 81 | + "nbconvert_exporter": "python", |
| 82 | + "pygments_lexer": "ipython3", |
| 83 | + "version": "3.6.3" |
| 84 | + } |
| 85 | + }, |
| 86 | + "nbformat": 4, |
| 87 | + "nbformat_minor": 2 |
| 88 | +} |
0 commit comments