From 899568311fd797dafe53414b72c8baf1d602bdd7 Mon Sep 17 00:00:00 2001 From: Jarek Zgoda Date: Fri, 25 Mar 2016 16:41:49 +0100 Subject: [PATCH 1/3] Use Treq for Twisted integration --- rollbar/__init__.py | 95 +++++++++------------------------------------ 1 file changed, 18 insertions(+), 77 deletions(-) diff --git a/rollbar/__init__.py b/rollbar/__init__.py index 7310d42e..7d9955c3 100644 --- a/rollbar/__init__.py +++ b/rollbar/__init__.py @@ -93,59 +93,8 @@ def wrap(*args, **kwargs): TornadoAsyncHTTPClient = None try: - from twisted.internet import reactor - from twisted.internet.defer import inlineCallbacks, Deferred, returnValue, succeed - from twisted.internet.protocol import Protocol + import treq from twisted.python import log as twisted_log - from twisted.web.client import Agent as TwistedHTTPClient - from twisted.web.http_headers import Headers as TwistedHeaders - from twisted.web.iweb import IBodyProducer - - from zope.interface import implementer - - - try: - # Verify we can make HTTPS requests with Twisted. - # From http://twistedmatrix.com/documents/12.0.0/core/howto/ssl.html - from OpenSSL import SSL - except ImportError: - log.exception('Rollbar requires SSL to work with Twisted') - raise - - - @implementer(IBodyProducer) - class StringProducer(object): - - def __init__(self, body): - self.body = body - self.length = len(body) - - def startProducing(self, consumer): - consumer.write(self.body) - return succeed(None) - - def pauseProducing(self): - pass - - def stopProducing(self): - pass - - - class ResponseAccumulator(Protocol): - def __init__(self, length, finished): - self.remaining = length - self.finished = finished - self.response = '' - - def dataReceived(self, bytes): - if self.remaining: - chunk = bytes[:self.remaining] - self.response += chunk - self.remaining -= len(chunk) - - def connectionLost(self, reason): - self.finished.callback(self.response) - def log_handler(event): """ @@ -171,9 +120,7 @@ def log_handler(event): except ImportError: - TwistedHTTPClient = None - inlineCallbacks = passthrough_decorator - StringProducer = None + treq = None def get_request(): @@ -450,8 +397,8 @@ def send_payload(payload, access_token): return _send_payload_appengine(payload, access_token) elif handler == 'twisted': - if TwistedHTTPClient is None: - log.error('Unable to find twisted') + if treq is None: + log.error('Unable to find Treq') return _send_payload_twisted(payload, access_token) else: @@ -1246,31 +1193,25 @@ def _send_payload_twisted(payload, access_token): log.exception('Exception while posting item %r', e) -@inlineCallbacks + def _post_api_twisted(path, payload, access_token=None): - headers = {'Content-Type': ['application/json']} + def post_data_cb(data, resp): + resp._content = data + _parse_response(path, settings['access_token'], payload, resp) + + def post_cb(resp): + r = requests.Response() + r.status_code = resp.code + r.headers.update(resp.headers.getAllRawHeaders()) + return treq.content(response).addCallback(post_data_cb, r) + + headers = {'Content-Type': ['application/json']} if access_token is not None: headers['X-Rollbar-Access-Token'] = [access_token] - url = urljoin(SETTINGS['endpoint'], path) - - agent = TwistedHTTPClient(reactor, connectTimeout=SETTINGS.get('timeout', DEFAULT_TIMEOUT)) - resp = yield agent.request( - 'POST', - url, - TwistedHeaders(headers), - StringProducer(payload)) - - r = requests.Response() - r.status_code = resp.code - r.headers.update(resp.headers.getAllRawHeaders()) - bodyDeferred = Deferred() - resp.deliverBody(ResponseAccumulator(resp.length, bodyDeferred)) - body = yield bodyDeferred - r._content = body - _parse_response(path, SETTINGS['access_token'], payload, r) - yield returnValue(None) + d = treq.post(url, payload, headers=headers, timeout=SETTINGS.get('timeout', DEFAULT_TIMEOUT)) + d.addCallback(post_cb) def _parse_response(path, access_token, params, resp, endpoint=None): From 75317ed502f2475e3ad7a1ec12ccce49afb7add9 Mon Sep 17 00:00:00 2001 From: Eric Zarowny Date: Mon, 11 Apr 2016 10:04:43 -0700 Subject: [PATCH 2/3] Tests and small fixes for Twisted --- .travis.yml | 2 +- rollbar/__init__.py | 10 +-- rollbar/examples/twisted/simpleserv.py | 4 +- .../test/{contrib => flask_tests}/__init__.py | 0 .../flask => flask_tests}/test_flask.py | 0 .../flask => twisted_tests}/__init__.py | 0 rollbar/test/twisted_tests/test_twisted.py | 88 +++++++++++++++++++ 7 files changed, 96 insertions(+), 8 deletions(-) rename rollbar/test/{contrib => flask_tests}/__init__.py (100%) rename rollbar/test/{contrib/flask => flask_tests}/test_flask.py (100%) rename rollbar/test/{contrib/flask => twisted_tests}/__init__.py (100%) create mode 100644 rollbar/test/twisted_tests/test_twisted.py diff --git a/.travis.yml b/.travis.yml index 623c0e66..777eca67 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ matrix: install: - if [ -v FLASK_VERSION ]; then pip install Flask==$FLASK_VERSION; fi - - if [ -v TWISTED_VERSION ]; then pip install Twisted==$TWISTED_VERSION service_identity pyOpenSSL; fi + - if [ -v TWISTED_VERSION ]; then pip install Twisted==$TWISTED_VERSION treq; fi - if [ -v DJANGO_VERSION ]; then pip install Django==$DJANGO_VERSION; fi script: diff --git a/rollbar/__init__.py b/rollbar/__init__.py index 7d9955c3..e253d64b 100644 --- a/rollbar/__init__.py +++ b/rollbar/__init__.py @@ -1193,24 +1193,24 @@ def _send_payload_twisted(payload, access_token): log.exception('Exception while posting item %r', e) - def _post_api_twisted(path, payload, access_token=None): - def post_data_cb(data, resp): resp._content = data - _parse_response(path, settings['access_token'], payload, resp) + _parse_response(path, SETTINGS['access_token'], payload, resp) def post_cb(resp): r = requests.Response() r.status_code = resp.code r.headers.update(resp.headers.getAllRawHeaders()) - return treq.content(response).addCallback(post_data_cb, r) + return treq.content(resp).addCallback(post_data_cb, r) headers = {'Content-Type': ['application/json']} if access_token is not None: headers['X-Rollbar-Access-Token'] = [access_token] + url = urljoin(SETTINGS['endpoint'], path) - d = treq.post(url, payload, headers=headers, timeout=SETTINGS.get('timeout', DEFAULT_TIMEOUT)) + d = treq.post(url, payload, headers=headers, + timeout=SETTINGS.get('timeout', DEFAULT_TIMEOUT)) d.addCallback(post_cb) diff --git a/rollbar/examples/twisted/simpleserv.py b/rollbar/examples/twisted/simpleserv.py index 488a5f29..2ba1d283 100644 --- a/rollbar/examples/twisted/simpleserv.py +++ b/rollbar/examples/twisted/simpleserv.py @@ -24,7 +24,7 @@ def foo(): class Echo(protocol.Protocol): """This is just about the simplest possible protocol""" - + def dataReceived(self, data): "As soon as any data is received, write it back." @@ -39,7 +39,7 @@ def main(): """This runs the protocol on port 8000""" factory = protocol.ServerFactory() factory.protocol = Echo - reactor.listenTCP(8000,factory) + reactor.listenTCP(8000, factory) reactor.run() diff --git a/rollbar/test/contrib/__init__.py b/rollbar/test/flask_tests/__init__.py similarity index 100% rename from rollbar/test/contrib/__init__.py rename to rollbar/test/flask_tests/__init__.py diff --git a/rollbar/test/contrib/flask/test_flask.py b/rollbar/test/flask_tests/test_flask.py similarity index 100% rename from rollbar/test/contrib/flask/test_flask.py rename to rollbar/test/flask_tests/test_flask.py diff --git a/rollbar/test/contrib/flask/__init__.py b/rollbar/test/twisted_tests/__init__.py similarity index 100% rename from rollbar/test/contrib/flask/__init__.py rename to rollbar/test/twisted_tests/__init__.py diff --git a/rollbar/test/twisted_tests/test_twisted.py b/rollbar/test/twisted_tests/test_twisted.py new file mode 100644 index 00000000..321a488d --- /dev/null +++ b/rollbar/test/twisted_tests/test_twisted.py @@ -0,0 +1,88 @@ +""" +Tests for Twisted instrumentation +""" + +import json +import sys + +import mock +import rollbar + +# access token for https://rollbar.com/rollbar/pyrollbar +TOKEN = '92c10f5616944b81a2e6f3c6493a0ec2' + +# Twisted hasn't been fully ported to Python 3 yet, so don't test there. +ALLOWED_PYTHON_VERSION = not sys.version_info[0] == 3 +try: + from twisted.test import proto_helpers + from twisted.trial import unittest + from twisted.internet import protocol + from twisted.python import log + + TWISTED_INSTALLED = True +except ImportError: + TWISTED_INSTALLED = False + +if ALLOWED_PYTHON_VERSION and TWISTED_INSTALLED: + class SquareProtocol(protocol.Protocol): + def dataReceived(self, data): + try: + number = int(data) + except ValueError: + log.err() + self.transport.write('error') + else: + self.transport.write(str(number**2)) + + class SquareFactory(protocol.Factory): + protocol = SquareProtocol + + class TwistedTest(unittest.TestCase): + def setUp(self): + rollbar.init(TOKEN, 'twisted-test') + factory = SquareFactory() + self.proto = factory.buildProtocol(('127.0.0.1', 0)) + self.tr = proto_helpers.StringTransport() + self.proto.makeConnection(self.tr) + + @mock.patch('rollbar.send_payload') + def test_base_case(self, send_payload): + self.proto.dataReceived('8') + self.assertEqual(int(self.tr.value()), 64) + + self.assertEqual(send_payload.called, False) + + @mock.patch('rollbar.send_payload') + def test_caught_exception(self, send_payload): + self.proto.dataReceived('rollbar') + self.assertEqual(self.tr.value(), "error") + errors = self.flushLoggedErrors(ValueError) + self.assertEqual(len(errors), 1) + + self.assertEqual(send_payload.called, True) + payload = json.loads(send_payload.call_args[0][0]) + data = payload['data'] + + self.assertIn('body', data) + self.assertEqual(data['body']['trace']['exception']['class'], + 'ValueError') + self.assertEqual(data['body']['trace']['exception']['message'], + "invalid literal for int() with base 10: 'rollbar'") + + # XXX not able to test uncaught exceptions for some reason + # @mock.patch('rollbar.send_payload') + # def test_uncaught_exception(self, send_payload): + # self.proto.dataReceived([8, 9]) + # self.assertEqual(self.tr.value(), "error") + # errors = self.flushLoggedErrors(TypeError) + # self.assertEqual(len(errors), 1) + # + # self.assertEqual(send_payload.called, True) + # payload = json.loads(send_payload.call_args[0][0]) + # data = payload['data'] + # + # self.assertIn('body', data) + # self.assertEqual(data['body']['trace']['exception']['class'], + # 'TypeError') + # self.assertEqual(data['body']['trace']['exception']['message'], + # "int() argument must be a string or a number, not 'list'") From 02273d280272206f6768815bafa5f64b28ad31b9 Mon Sep 17 00:00:00 2001 From: Eric Zarowny Date: Mon, 18 Apr 2016 10:38:06 -0700 Subject: [PATCH 3/3] update versions of Twisted on TravisCI --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 777eca67..ae4dacf0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,9 @@ matrix: env: FLASK_VERSION=0.10.1 - python: "2.7" - env: TWISTED_VERSION=15.4 + env: TWISTED_VERSION=15.5.0 + - python: "2.7" + env: TWISTED_VERSION=16.1.1 - python: "2.6" env: DJANGO_VERSION=1.4