From b1169d7f5e564bae85f0026af0e58d2457d56ae2 Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Thu, 29 Dec 2016 22:12:09 -0600 Subject: [PATCH 01/44] Switch Boto2 to Boto3 for SQS messaging --- kombu/async/aws/ext.py | 13 ++- kombu/async/aws/sqs/ext.py | 5 +- kombu/transport/SQS.py | 94 +++++++++-------- requirements/extras/sqs.txt | 1 + requirements/funtest.txt | 1 + t/unit/transport/test_SQS.py | 192 +++++++++++++++++------------------ 6 files changed, 168 insertions(+), 138 deletions(-) diff --git a/kombu/async/aws/ext.py b/kombu/async/aws/ext.py index b0e497cbe..e645f9736 100644 --- a/kombu/async/aws/ext.py +++ b/kombu/async/aws/ext.py @@ -1,8 +1,11 @@ # -*- coding: utf-8 -*- -"""Amazon boto interface.""" +"""Amazon boto3 interface.""" from __future__ import absolute_import, unicode_literals try: + import boto3 + + # TODO: old.. import boto except ImportError: # pragma: no cover boto = get_regions = ResultSet = RegionInfo = XmlHandler = None @@ -17,6 +20,11 @@ class BotoError(Exception): exception.SQSError = BotoError exception.SQSDecodeError = BotoError else: + from botocore import exceptions + # from boto3 import exceptions + from boto3 import session + + # TODO: old.. from boto import exception from boto.connection import AWSAuthConnection, AWSQueryConnection from boto.handler import XmlHandler @@ -24,6 +32,9 @@ class BotoError(Exception): from boto.regioninfo import RegionInfo, get_regions __all__ = [ + 'exceptions', + + # TODO: old.. 'exception', 'AWSAuthConnection', 'AWSQueryConnection', 'XmlHandler', 'ResultSet', 'RegionInfo', 'get_regions', ] diff --git a/kombu/async/aws/sqs/ext.py b/kombu/async/aws/sqs/ext.py index eb48f3e95..064155390 100644 --- a/kombu/async/aws/sqs/ext.py +++ b/kombu/async/aws/sqs/ext.py @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- -"""Amazon SQS boto interface.""" +"""Amazon SQS boto3 interface.""" from __future__ import absolute_import, unicode_literals try: + import boto3 + + # TODO: old.. import boto except ImportError: # pragma: no cover boto = Attributes = BatchResults = None # noqa diff --git a/kombu/transport/SQS.py b/kombu/transport/SQS.py index 1e8ef6db1..721b3ed39 100644 --- a/kombu/transport/SQS.py +++ b/kombu/transport/SQS.py @@ -44,6 +44,9 @@ from kombu.async import get_event_loop from kombu.async.aws import sqs as _asynsqs +from kombu.async.aws.ext import boto3, exceptions + +# TODO: old.. from kombu.async.aws.ext import boto, exception from kombu.async.aws.sqs.connection import AsyncSQSConnection, SQSConnection from kombu.async.aws.sqs.ext import regions @@ -104,18 +107,24 @@ def __init__(self, *args, **kwargs): self.hub = kwargs.get('hub') or get_event_loop() def _update_queue_cache(self, queue_name_prefix): - try: - queues = self.sqs.get_all_queues(prefix=queue_name_prefix) - except exception.SQSError as exc: - if exc.status == 403: - raise RuntimeError( - 'SQS authorization error, access_key={0}'.format( - self.sqs.access_key)) - raise - else: - self._queue_cache.update({ - queue.name: queue for queue in queues - }) + for q in self.sqs.queues.filter(QueueNamePrefix=queue_name_prefix): + name = q.url.split('/')[-1] + self._queue_cache[name] = q + # TODO: Old boto 2 code. Is this error patching needed? + # try: + # queues = self.sqs.get_all_queues(prefix=queue_name_prefix) + # except exceptions.BotoCoreError as exc: + # raise + # except exception.SQSError as exc: + # if exc.status == 403: + # raise RuntimeError( + # 'SQS authorization error, access_key={0}'.format( + # self.sqs.access_key)) + # raise + # else: + # self._queue_cache.update({ + # queue.name: queue for queue in queues + # }) def basic_consume(self, queue, no_ack, *args, **kwargs): if no_ack: @@ -132,7 +141,7 @@ def basic_cancel(self, consumer_tag): self._noack_queues.discard(queue) return super(Channel, self).basic_cancel(consumer_tag) - def drain_events(self, timeout=None): + def drain_events(self, timeout=None, callback=None): """Return a single payload message from one of our queues. Raises: @@ -143,7 +152,7 @@ def drain_events(self, timeout=None): raise Empty() # At this point, go and get more messages from SQS - self._poll(self.cycle, self.connection._deliver, timeout=timeout) + self._poll(self.cycle, callback, timeout=timeout) def _reset_cycle(self): """Reset the consume cycle. @@ -192,14 +201,12 @@ def _delete(self, queue, *args, **kwargs): def _put(self, queue, message, **kwargs): """Put message onto queue.""" q = self._new_queue(queue) - m = Message() - m.set_body(dumps(message)) - q.write(m) + q.send_message(MessageBody=dumps(message)) def _message_to_python(self, message, queue_name, queue): - payload = loads(bytes_to_str(message.get_body())) + payload = loads(bytes_to_str(message.body)) if queue_name in self._noack_queues: - queue.delete_message(message) + message.delete() else: try: properties = payload['properties'] @@ -209,7 +216,7 @@ def _message_to_python(self, message, queue_name, queue): delivery_info = {} properties = {'delivery_info': delivery_info} payload.update({ - 'body': bytes_to_str(message.get_body()), + 'body': bytes_to_str(message.body), 'properties': properties, }) # set delivery tag to SQS receipt handle @@ -261,10 +268,12 @@ def _get_bulk(self, queue, # drain_events calls `can_consume` first, consuming # a token, so we know that we are allowed to consume at least # one message. - maxcount = self._get_message_estimate() - if maxcount: + + # Note: ignoring max_messages for SQS with boto3 + max_count = self._get_message_estimate() + if max_count: q = self._new_queue(queue) - messages = q.get_messages(num_messages=maxcount) + messages = q.receive_messages(MaxNumberOfMessages=max_count) if messages: for msg in self._messages_to_python(messages, queue): @@ -275,7 +284,8 @@ def _get_bulk(self, queue, def _get(self, queue): """Try to retrieve a single message off ``queue``.""" q = self._new_queue(queue) - messages = q.get_messages(num_messages=1) + messages = q.receive_messages() + if messages: return self._messages_to_python(messages, queue)[0] raise Empty() @@ -344,18 +354,17 @@ def _restore(self, message, return super(Channel, self)._restore(message) def basic_ack(self, delivery_tag, multiple=False): - delivery_info = self.qos.get(delivery_tag).delivery_info try: - queue = delivery_info['sqs_queue'] + message = self.qos.get(delivery_tag).delivery_info['sqs_message'] except KeyError: pass else: - queue.delete_message(delivery_info['sqs_message']) + message.delete() super(Channel, self).basic_ack(delivery_tag) def _size(self, queue): """Return the number of messages in a queue.""" - return self._new_queue(queue).count() + return int(self._new_queue(queue).attributes['ApproximateNumberOfMessages']) def _purge(self, queue): """Delete all current messages in a queue.""" @@ -364,21 +373,20 @@ def _purge(self, queue): # iterations to ensure messages are deleted. size = 0 for i in range(10): - size += q.count() + size += int(q.attributes['ApproximateNumberOfMessages']) if not size: break - q.clear() + q.purge() return size def close(self): super(Channel, self).close() - for conn in (self._sqs, self._asynsqs): - if conn: - try: - conn.close() - except AttributeError as exc: # FIXME ??? - if "can't set attribute" not in str(exc): - raise + if self._asynsqs: + try: + self.asynsqs.close() + except AttributeError as exc: # FIXME ??? + if "can't set attribute" not in str(exc): + raise def _get_regioninfo(self, regions): if self.regioninfo: @@ -402,8 +410,14 @@ def _aws_connect_to(self, fun, regions): @property def sqs(self): if self._sqs is None: - self._sqs = self._aws_connect_to(SQSConnection, regions()) + # TODO: Update this to support region setting.. + session = boto3.session.Session() + self._sqs = session.resource('sqs') return self._sqs + # TODO: old boto 2 code (SQSConnection is a boto 2 thing) + # if self._sqs is None: + # self._sqs = self._aws_connect_to(SQSConnection, regions()) + # return self._sqs @property def asynsqs(self): @@ -466,10 +480,10 @@ class Transport(virtual.Transport): default_port = None connection_errors = ( virtual.Transport.connection_errors + - (exception.SQSError, socket.error) + (exceptions.BotoCoreError, socket.error) ) channel_errors = ( - virtual.Transport.channel_errors + (exception.SQSDecodeError,) + virtual.Transport.channel_errors + (exceptions.BotoCoreError,) ) driver_type = 'sqs' driver_name = 'sqs' diff --git a/requirements/extras/sqs.txt b/requirements/extras/sqs.txt index 53851e77c..21046fbd6 100644 --- a/requirements/extras/sqs.txt +++ b/requirements/extras/sqs.txt @@ -1,2 +1,3 @@ boto>=2.8 +boto3 pycurl diff --git a/requirements/funtest.txt b/requirements/funtest.txt index 556373090..033db4b8b 100644 --- a/requirements/funtest.txt +++ b/requirements/funtest.txt @@ -9,6 +9,7 @@ kazoo # SQS transport boto +boto3 # Qpid transport qpid-python>=0.26 diff --git a/t/unit/transport/test_SQS.py b/t/unit/transport/test_SQS.py index f245fdab6..971f8a55f 100644 --- a/t/unit/transport/test_SQS.py +++ b/t/unit/transport/test_SQS.py @@ -19,84 +19,82 @@ from kombu.transport import SQS +class SQSMessageMock(object): + def __init__(self): + """ + Imitate the SQS Message from boto3. + """ + self.body = "" + self.receipt_handle = "receipt_handle_xyz" + + class SQSQueueMock(object): + """ + Imitate the SQS Queue in boto3. It has two attributes, url and + "attributes". + """ - def __init__(self, name): - self.name = name - self.messages = [] - self._get_message_calls = 0 + def __init__(self, url): + self.url = url + self.attributes = {'ApproximateNumberOfMessages': '0'} - def clear(self, page_size=10, vtimeout=10): + self.messages = [] + self._receive_messages_calls = 0 + + def send_message(self, MessageBody=None, DelaySeconds=0, MessageAttributes=None): + msg = SQSMessageMock() + msg.body = MessageBody + self.messages.append(msg) + self.attributes['ApproximateNumberOfMessages'] = len(self.messages) + + def receive_messages(self, MaxNumberOfMessages=10): + self._receive_messages_calls += 1 + returning = self.messages[:MaxNumberOfMessages] + self.messages = self.messages[MaxNumberOfMessages:] + self.attributes['ApproximateNumberOfMessages'] = len(self.messages) + return returning + + def purge(self): empty, self.messages[:] = not self.messages, [] + self.attributes['ApproximateNumberOfMessages'] = len(self.messages) return not empty - def count(self, page_size=10, vtimeout=10): - return len(self.messages) - count_slow = count - - def delete(self): - self.messages[:] = [] - return True - - def delete_message(self, message): - try: - self.messages.remove(message) - except ValueError: - return False - return True - - def get_messages(self, num_messages=1, visibility_timeout=None, - attributes=None, *args, **kwargs): - self._get_message_calls += 1 - messages, self.messages[:num_messages] = ( - self.messages[:num_messages], []) - return messages - - def read(self, visibility_timeout=None): - return self.messages.pop(0) - - def write(self, message): - self.messages.append(message) - return True class SQSConnectionMock(object): def __init__(self): - self.queues = { + """ + Imitate the SQS Resource in boto3. + """ + self.queues = self.Queues() + self.queues.queues = { 'q_%s' % n: SQSQueueMock('q_%s' % n) for n in range(1500) } q = SQSQueueMock('unittest_queue') - q.write('hello') - self.queues['unittest_queue'] = q - - def get_queue(self, queue): - return self.queues.get(queue) - - def get_all_queues(self, prefix=""): - if not prefix: - keys = sorted(self.queues.keys())[:1000] - else: - keys = list(filter( - lambda k: k.startswith(prefix), sorted(self.queues.keys()) - ))[:1000] - return [self.queues[key] for key in keys] - - def delete_queue(self, queue, force_deletion=False): - q = self.get_queue(queue) - if q: - if q.count(): - return False - q.clear() - self.queues.pop(queue, None) - - def delete_message(self, queue, message): - return queue.delete_message(message) - - def create_queue(self, name, *args, **kwargs): - q = self.queues[name] = SQSQueueMock(name) + q.send_message(MessageBody='hello') + self.queues.queues['unittest_queue'] = q + + class Queues: + """ + Imitate the queues attribute on the SQS Resource in boto3. It has + a filter() method and is an iterator rather than a list. + """ + def __init__(self): + self.queues = [] + + def filter(self, QueueNamePrefix=None): + """ Return an iterable of queues. """ + return (val for key, val in self.queues.items() + if key.startswith(QueueNamePrefix)) + + def create_queue(self, QueueName=None, Attributes=None): + q = self.queues.queues[QueueName] = SQSQueueMock(QueueName) return q + def get_queue_by_name(self, QueueName=None): + return self.queues.queues.get(QueueName) + @skip.unless_module('boto') class test_Channel: @@ -160,33 +158,35 @@ def test_init(self): """kombu.SQS.Channel instantiates correctly with mocked queues""" assert self.queue_name in self.channel._queue_cache - def test_auth_fail(self): - normal_func = SQS.Channel.sqs.get_all_queues - - def get_all_queues_fail_403(prefix=''): - # mock auth error - raise exception.SQSError(403, None, None) - - def get_all_queues_fail_not_403(prefix=''): - # mock non-auth error - raise exception.SQSError(500, None, None) - - try: - SQS.Channel.sqs.access_key = '1234' - SQS.Channel.sqs.get_all_queues = get_all_queues_fail_403 - with pytest.raises(RuntimeError) as excinfo: - self.channel = self.connection.channel() - assert 'access_key=1234' in str(excinfo.value) - SQS.Channel.sqs.get_all_queues = get_all_queues_fail_not_403 - with pytest.raises(exception.SQSError): - self.channel = self.connection.channel() - finally: - SQS.Channel.sqs.get_all_queues = normal_func + # TODO: I dropped the old behavior of changing the error message when + # bad auth is provided. Is this stuff important? + # def test_auth_fail(self): + # normal_func = SQS.Channel.sqs.list_queues + # + # def get_all_queues_fail_403(prefix=''): + # # mock auth error + # raise exception.SQSError(403, None, None) + # + # def get_all_queues_fail_not_403(prefix=''): + # # mock non-auth error + # raise exception.SQSError(500, None, None) + # + # try: + # SQS.Channel.sqs.access_key = '1234' + # SQS.Channel.sqs.get_all_queues = get_all_queues_fail_403 + # with pytest.raises(RuntimeError) as excinfo: + # self.channel = self.connection.channel() + # assert 'access_key=1234' in str(excinfo.value) + # SQS.Channel.sqs.get_all_queues = get_all_queues_fail_not_403 + # with pytest.raises(exception.SQSError): + # self.channel = self.connection.channel() + # finally: + # SQS.Channel.sqs.get_all_queues = normal_func def test_new_queue(self): queue_name = 'new_unittest_queue' self.channel._new_queue(queue_name) - assert queue_name in self.sqs_conn_mock.queues + assert queue_name in self.sqs_conn_mock.queues.queues # For cleanup purposes, delete the queue and the queue file self.channel._delete(queue_name) @@ -196,10 +196,10 @@ def test_dont_create_duplicate_new_queue(self): # first 1000 queues sorted by name. queue_name = 'unittest_queue' self.channel._new_queue(queue_name) - assert queue_name in self.sqs_conn_mock.queues - q = self.sqs_conn_mock.get_queue(queue_name) - assert 1 == q.count() - assert 'hello' == q.read() + assert queue_name in self.sqs_conn_mock.queues.queues + queue = self.sqs_conn_mock.queues.queues[queue_name] + assert 1 == int(queue.attributes['ApproximateNumberOfMessages']) + assert 'hello' == queue.messages[0].body def test_delete(self): queue_name = 'new_unittest_queue' @@ -212,7 +212,7 @@ def test_get_from_sqs(self): message = 'my test message' self.producer.publish(message) q = self.channel._new_queue(self.queue_name) - results = q.get_messages() + results = q.receive_messages() assert len(results) == 1 # Now test getting many messages @@ -220,7 +220,7 @@ def test_get_from_sqs(self): message = 'message: {0}'.format(i) self.producer.publish(message) - results = q.get_messages(num_messages=3) + results = q.receive_messages(MaxNumberOfMessages=3) assert len(results) == 3 def test_get_with_empty_list(self): @@ -246,8 +246,8 @@ def test_messages_to_python(self): q = self.channel._new_queue(self.queue_name) # Get the messages now - kombu_messages = q.get_messages(num_messages=kombu_message_count) - json_messages = q.get_messages(num_messages=json_message_count) + kombu_messages = q.receive_messages(MaxNumberOfMessages=kombu_message_count) + json_messages = q.receive_messages(MaxNumberOfMessages=json_message_count) # Now convert them to payloads kombu_payloads = self.channel._messages_to_python( @@ -350,7 +350,7 @@ def on_message_delivered(message, queue): def test_drain_events_with_prefetch_none(self): # Generate 20 messages message_count = 20 - expected_get_message_count = 3 + expected_receive_messages_count = 3 current_delivery_tag = [1] @@ -379,5 +379,5 @@ def on_message_delivered(message, queue): assert self.channel.connection._deliver.call_count == message_count # How many times was the SQSConnectionMock get_message method called? - assert (expected_get_message_count == - self.channel._queue_cache[self.queue_name]._get_message_calls) + assert (expected_receive_messages_count == + self.channel._queue_cache[self.queue_name]._receive_messages_calls) From fad9250f68f26e6e43754a6f8e88424bf5d9f149 Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Fri, 30 Dec 2016 11:52:28 -0600 Subject: [PATCH 02/44] Fixed region support --- kombu/transport/SQS.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/kombu/transport/SQS.py b/kombu/transport/SQS.py index 721b3ed39..18e990007 100644 --- a/kombu/transport/SQS.py +++ b/kombu/transport/SQS.py @@ -189,7 +189,8 @@ def _new_queue(self, queue, **kwargs): return self._queue_cache[queue] except KeyError: q = self._queue_cache[queue] = self.sqs.create_queue( - queue, self.visibility_timeout, + QueueName=queue, + Attributes={'VisibilityTimeout': str(self.visibility_timeout)} ) return q @@ -410,9 +411,16 @@ def _aws_connect_to(self, fun, regions): @property def sqs(self): if self._sqs is None: - # TODO: Update this to support region setting.. - session = boto3.session.Session() - self._sqs = session.resource('sqs') + # TODO: Move to _aws_connect_to once the async stuff uses boto3 + is_secure = self.is_secure if self.is_secure is not None else True + # Note: port number is not supported by boto3 + # port = self.port if self.port is not None else self.conninfo.port + session = boto3.session.Session( + region_name=self.region, + aws_access_key_id=self.conninfo.userid, + aws_secret_access_key=self.conninfo.password + ) + self._sqs = session.resource('sqs', use_ssl=is_secure) return self._sqs # TODO: old boto 2 code (SQSConnection is a boto 2 thing) # if self._sqs is None: From 39b7bd3a2f1371d79721a520614b685d0d3286ed Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Fri, 30 Dec 2016 12:30:01 -0600 Subject: [PATCH 03/44] Add SQS FIFO queue support --- kombu/transport/SQS.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/kombu/transport/SQS.py b/kombu/transport/SQS.py index 18e990007..c2db89a5a 100644 --- a/kombu/transport/SQS.py +++ b/kombu/transport/SQS.py @@ -169,7 +169,12 @@ def _reset_cycle(self): def entity_name(self, name, table=CHARS_REPLACE_TABLE): """Format AMQP queue name into a legal SQS queue name.""" - return text_t(safe_str(name)).translate(table) + if name.endswith('.fifo'): + partial = name.rstrip('.fifo') + partial = text_t(safe_str(partial)).translate(table) + return partial + '.fifo' + else: + return text_t(safe_str(name)).translate(table) def _new_queue(self, queue, **kwargs): """Ensure a queue with given name exists in SQS.""" @@ -188,10 +193,12 @@ def _new_queue(self, queue, **kwargs): try: return self._queue_cache[queue] except KeyError: + attributes = {'VisibilityTimeout': str(self.visibility_timeout)} + if queue.endswith('.fifo'): + attributes['FifoQueue'] = 'true' + q = self._queue_cache[queue] = self.sqs.create_queue( - QueueName=queue, - Attributes={'VisibilityTimeout': str(self.visibility_timeout)} - ) + QueueName=queue, Attributes=attributes) return q def _delete(self, queue, *args, **kwargs): @@ -202,7 +209,12 @@ def _delete(self, queue, *args, **kwargs): def _put(self, queue, message, **kwargs): """Put message onto queue.""" q = self._new_queue(queue) - q.send_message(MessageBody=dumps(message)) + kwargs = {'MessageBody': dumps(message)} + if 'MessageGroupId' in message['properties']: + kwargs['MessageGroupId'] = message['properties']['MessageGroupId'] + if 'MessageDeduplicationId' in message['properties']: + kwargs['MessageDeduplicationId'] = message['properties']['MessageDeduplicationId'] + q.send_message(**kwargs) def _message_to_python(self, message, queue_name, queue): payload = loads(bytes_to_str(message.body)) From 04ee160a436a040092f711a5afdbffe7d2be04df Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Tue, 3 Jan 2017 11:21:07 -0600 Subject: [PATCH 04/44] Add sensible defaults for message attributes --- kombu/transport/SQS.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/kombu/transport/SQS.py b/kombu/transport/SQS.py index c2db89a5a..9e3b18b81 100644 --- a/kombu/transport/SQS.py +++ b/kombu/transport/SQS.py @@ -39,6 +39,7 @@ import socket import string +import uuid from vine import transform, ensure_promise, promise @@ -210,10 +211,15 @@ def _put(self, queue, message, **kwargs): """Put message onto queue.""" q = self._new_queue(queue) kwargs = {'MessageBody': dumps(message)} - if 'MessageGroupId' in message['properties']: - kwargs['MessageGroupId'] = message['properties']['MessageGroupId'] - if 'MessageDeduplicationId' in message['properties']: - kwargs['MessageDeduplicationId'] = message['properties']['MessageDeduplicationId'] + if queue.endswith('.fifo'): + if 'MessageGroupId' in message['properties']: + kwargs['MessageGroupId'] = message['properties']['MessageGroupId'] + else: + kwargs['MessageGroupId'] = 'default' + if 'MessageDeduplicationId' in message['properties']: + kwargs['MessageDeduplicationId'] = message['properties']['MessageDeduplicationId'] + else: + kwargs['MessageDeduplicationId'] = str(uuid.uuid4()) q.send_message(**kwargs) def _message_to_python(self, message, queue_name, queue): From c270e9223a4a21c12790578cb0c6f2a7a18265cf Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Thu, 12 Jan 2017 12:14:57 -0600 Subject: [PATCH 05/44] Asynchronous support, plus boto3 for region endpoint lookups --- kombu/async/aws/sqs/connection.py | 14 ++- kombu/async/aws/sqs/message.py | 23 ++-- kombu/transport/SQS.py | 117 ++++++++--------- t/unit/async/aws/sqs/test_message.py | 37 ------ t/unit/transport/test_SQS.py | 181 +++++++++++++-------------- 5 files changed, 156 insertions(+), 216 deletions(-) delete mode 100644 t/unit/async/aws/sqs/test_message.py diff --git a/kombu/async/aws/sqs/connection.py b/kombu/async/aws/sqs/connection.py index 32b9926bb..d73e7ea62 100644 --- a/kombu/async/aws/sqs/connection.py +++ b/kombu/async/aws/sqs/connection.py @@ -7,7 +7,7 @@ from kombu.async.aws.connection import AsyncAWSQueryConnection from kombu.async.aws.ext import RegionInfo -from .ext import boto, Attributes, BatchResults, SQSConnection +from .ext import boto, boto3, Attributes, BatchResults, SQSConnection from .message import AsyncMessage from .queue import AsyncQueue @@ -24,6 +24,8 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, https_connection_factory=None, region=None, *args, **kwargs): if boto is None: raise ImportError('boto is not installed') + if boto3 is None: + raise ImportError('boto3 is not installed') self.region = region or RegionInfo( self, self.DefaultRegionName, self.DefaultRegionEndpoint, connection_cls=type(self), @@ -78,13 +80,13 @@ def receive_message(self, queue, if wait_time_seconds is not None: params['WaitTimeSeconds'] = wait_time_seconds return self.get_list( - 'ReceiveMessage', params, [('Message', queue.message_class)], - queue.id, callback=callback, + 'ReceiveMessage', params, [('Message', AsyncMessage)], + queue, callback=callback, parent=queue, ) - def delete_message(self, queue, message, callback=None): + def delete_message(self, queue, receipt_handle, callback=None): return self.delete_message_from_handle( - queue, message.receipt_handle, callback, + queue, receipt_handle, callback, ) def delete_message_batch(self, queue, messages, callback=None): @@ -104,7 +106,7 @@ def delete_message_from_handle(self, queue, receipt_handle, callback=None): return self.get_status( 'DeleteMessage', {'ReceiptHandle': receipt_handle}, - queue.id, callback=callback, + queue, callback=callback, ) def send_message(self, queue, message_content, diff --git a/kombu/async/aws/sqs/message.py b/kombu/async/aws/sqs/message.py index 363598413..ed4a9060a 100644 --- a/kombu/async/aws/sqs/message.py +++ b/kombu/async/aws/sqs/message.py @@ -15,16 +15,6 @@ class BaseAsyncMessage(object): """Base class for messages received on async client.""" - def delete(self, callback=None): - if self.queue: - return self.queue.delete_message(self, callback) - - def change_visibility(self, visibility_timeout, callback=None): - if self.queue: - return self.queue.connection.change_message_visibility( - self.queue, self.receipt_handle, visibility_timeout, callback, - ) - class AsyncRawMessage(BaseAsyncMessage, RawMessage): """Raw Message.""" @@ -33,6 +23,19 @@ class AsyncRawMessage(BaseAsyncMessage, RawMessage): class AsyncMessage(BaseAsyncMessage, Message): """Serialized message.""" + def __getitem__(self, item): + """ + Support Boto3-style access on a message. + """ + if item == 'ReceiptHandle': + return self.receipt_handle + elif item == 'Body': + return self.get_body() + elif item == 'queue': + return self.queue + else: + raise KeyError(item) + class AsyncMHMessage(BaseAsyncMessage, MHMessage): """MHM Message (uhm, look that up later).""" diff --git a/kombu/transport/SQS.py b/kombu/transport/SQS.py index 9e3b18b81..1df36db88 100644 --- a/kombu/transport/SQS.py +++ b/kombu/transport/SQS.py @@ -39,6 +39,7 @@ import socket import string +from urllib.parse import urlparse, urlunparse import uuid from vine import transform, ensure_promise, promise @@ -48,9 +49,8 @@ from kombu.async.aws.ext import boto3, exceptions # TODO: old.. -from kombu.async.aws.ext import boto, exception -from kombu.async.aws.sqs.connection import AsyncSQSConnection, SQSConnection -from kombu.async.aws.sqs.ext import regions +from kombu.async.aws.ext import boto +from kombu.async.aws.sqs.connection import AsyncSQSConnection from kombu.async.aws.sqs.message import Message from kombu.five import Empty, range, string_t, text_t from kombu.log import get_logger @@ -108,24 +108,10 @@ def __init__(self, *args, **kwargs): self.hub = kwargs.get('hub') or get_event_loop() def _update_queue_cache(self, queue_name_prefix): - for q in self.sqs.queues.filter(QueueNamePrefix=queue_name_prefix): - name = q.url.split('/')[-1] - self._queue_cache[name] = q - # TODO: Old boto 2 code. Is this error patching needed? - # try: - # queues = self.sqs.get_all_queues(prefix=queue_name_prefix) - # except exceptions.BotoCoreError as exc: - # raise - # except exception.SQSError as exc: - # if exc.status == 403: - # raise RuntimeError( - # 'SQS authorization error, access_key={0}'.format( - # self.sqs.access_key)) - # raise - # else: - # self._queue_cache.update({ - # queue.name: queue for queue in queues - # }) + resp = self.sqs.list_queues(QueueNamePrefix=queue_name_prefix) + for url in resp.get('QueueUrls', []): + queue_name = url.split('/')[-1] + self._queue_cache[queue_name] = url def basic_consume(self, queue, no_ack, *args, **kwargs): if no_ack: @@ -198,9 +184,10 @@ def _new_queue(self, queue, **kwargs): if queue.endswith('.fifo'): attributes['FifoQueue'] = 'true' - q = self._queue_cache[queue] = self.sqs.create_queue( + resp = self._queue_cache[queue] = self.sqs.create_queue( QueueName=queue, Attributes=attributes) - return q + self._queue_cache[queue] = resp['QueueUrl'] + return resp['QueueUrl'] def _delete(self, queue, *args, **kwargs): """Delete queue by name.""" @@ -209,8 +196,8 @@ def _delete(self, queue, *args, **kwargs): def _put(self, queue, message, **kwargs): """Put message onto queue.""" - q = self._new_queue(queue) - kwargs = {'MessageBody': dumps(message)} + q_url = self._new_queue(queue) + kwargs = {'QueueUrl': q_url, 'MessageBody': Message().encode(dumps(message))} if queue.endswith('.fifo'): if 'MessageGroupId' in message['properties']: kwargs['MessageGroupId'] = message['properties']['MessageGroupId'] @@ -220,12 +207,13 @@ def _put(self, queue, message, **kwargs): kwargs['MessageDeduplicationId'] = message['properties']['MessageDeduplicationId'] else: kwargs['MessageDeduplicationId'] = str(uuid.uuid4()) - q.send_message(**kwargs) + self.sqs.send_message(**kwargs) def _message_to_python(self, message, queue_name, queue): - payload = loads(bytes_to_str(message.body)) + payload = loads(bytes_to_str(message['Body'])) if queue_name in self._noack_queues: - message.delete() + queue = self._new_queue(queue_name) + self.asynsqs.delete_message(queue, message['ReceiptHandle']) else: try: properties = payload['properties'] @@ -235,14 +223,14 @@ def _message_to_python(self, message, queue_name, queue): delivery_info = {} properties = {'delivery_info': delivery_info} payload.update({ - 'body': bytes_to_str(message.body), + 'body': bytes_to_str(message['Body']), 'properties': properties, }) # set delivery tag to SQS receipt handle delivery_info.update({ 'sqs_message': message, 'sqs_queue': queue, }) - properties['delivery_tag'] = message.receipt_handle + properties['delivery_tag'] = message['ReceiptHandle'] return payload def _messages_to_python(self, messages, queue): @@ -291,22 +279,25 @@ def _get_bulk(self, queue, # Note: ignoring max_messages for SQS with boto3 max_count = self._get_message_estimate() if max_count: - q = self._new_queue(queue) - messages = q.receive_messages(MaxNumberOfMessages=max_count) + q_url = self._new_queue(queue) + resp = self.sqs.receive_message(QueueUrl=q_url, MaxNumberOfMessages=max_count) - if messages: - for msg in self._messages_to_python(messages, queue): + if resp['Messages']: + for m in resp['Messages']: + m['Body'] = Message().decode(m['Body']) + for msg in self._messages_to_python(resp['Messages'], queue): self.connection._deliver(msg, queue) return raise Empty() def _get(self, queue): """Try to retrieve a single message off ``queue``.""" - q = self._new_queue(queue) - messages = q.receive_messages() + q_url = self._new_queue(queue) + resp = self.sqs.receive_message(q_url) - if messages: - return self._messages_to_python(messages, queue)[0] + if resp['Messages']: + resp['Messages'][0]['Body'] = Message().decode(resp['Messages'][0]['Body']) + return self._messages_to_python(resp['Messages'], queue)[0] raise Empty() def _loop1(self, queue, _=None): @@ -340,8 +331,12 @@ def _get_bulk_async(self, queue, def _get_async(self, queue, count=1, callback=None): q = self._new_queue(queue) + result = list(urlparse(q)) + result[0] = result[1] = '' + q_shortpath = urlunparse(result) + return self._get_from_sqs( - q, count=count, connection=self.asynsqs, + q_shortpath, count=count, connection=self.asynsqs, callback=transform(self._on_messages_ready, callback, q, queue), ) @@ -378,24 +373,26 @@ def basic_ack(self, delivery_tag, multiple=False): except KeyError: pass else: - message.delete() + resp = self.asynsqs.delete_message(message['queue'], message['ReceiptHandle']) super(Channel, self).basic_ack(delivery_tag) def _size(self, queue): """Return the number of messages in a queue.""" - return int(self._new_queue(queue).attributes['ApproximateNumberOfMessages']) + url = self._new_queue(queue) + resp = self.sqs.get_queue_attributes(QueueUrl=url, AttributeNames=['ApproximateNumberOfMessages']) + return int(resp['Attributes']['ApproximateNumberOfMessages']) def _purge(self, queue): """Delete all current messages in a queue.""" q = self._new_queue(queue) # SQS is slow at registering messages, so run for a few - # iterations to ensure messages are deleted. + # iterations to ensure messages are detected and deleted. size = 0 for i in range(10): - size += int(q.attributes['ApproximateNumberOfMessages']) + size += int(self._size(queue)) if not size: break - q.purge() + self.sqs.purge_queue(q) return size def close(self): @@ -415,41 +412,27 @@ def _get_regioninfo(self, regions): if _r.name == self.region: return _r - def _aws_connect_to(self, fun, regions): - conninfo = self.conninfo - region = self._get_regioninfo(regions) - is_secure = self.is_secure if self.is_secure is not None else True - port = self.port if self.port is not None else conninfo.port - return fun(region=region, - aws_access_key_id=conninfo.userid, - aws_secret_access_key=conninfo.password, - is_secure=is_secure, - port=port) - @property def sqs(self): if self._sqs is None: - # TODO: Move to _aws_connect_to once the async stuff uses boto3 - is_secure = self.is_secure if self.is_secure is not None else True - # Note: port number is not supported by boto3 - # port = self.port if self.port is not None else self.conninfo.port session = boto3.session.Session( region_name=self.region, aws_access_key_id=self.conninfo.userid, - aws_secret_access_key=self.conninfo.password + aws_secret_access_key=self.conninfo.password, ) - self._sqs = session.resource('sqs', use_ssl=is_secure) + is_secure = self.is_secure if self.is_secure is not None else True + self._sqs = session.client('sqs', use_ssl=is_secure) return self._sqs - # TODO: old boto 2 code (SQSConnection is a boto 2 thing) - # if self._sqs is None: - # self._sqs = self._aws_connect_to(SQSConnection, regions()) - # return self._sqs @property def asynsqs(self): if self._asynsqs is None: - self._asynsqs = self._aws_connect_to( - AsyncSQSConnection, _asynsqs.regions(), + region = self._get_regioninfo(_asynsqs.regions()) + self._asynsqs = AsyncSQSConnection( + aws_access_key_id=self.conninfo.userid, + aws_secret_access_key=self.conninfo.password, + region=region, + is_secure=self.is_secure if self.is_secure is not None else True, ) return self._asynsqs diff --git a/t/unit/async/aws/sqs/test_message.py b/t/unit/async/aws/sqs/test_message.py deleted file mode 100644 index 44a0ac32d..000000000 --- a/t/unit/async/aws/sqs/test_message.py +++ /dev/null @@ -1,37 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import, unicode_literals - -from case import Mock - -from kombu.async.aws.sqs.message import AsyncMessage -from kombu.utils.uuid import uuid - -from t.mocks import PromiseMock - -from ..case import AWSCase - - -class test_AsyncMessage(AWSCase): - - def setup(self): - self.queue = Mock(name='queue') - self.callback = PromiseMock(name='callback') - self.x = AsyncMessage(self.queue, 'body') - self.x.receipt_handle = uuid() - - def test_delete(self): - assert self.x.delete(callback=self.callback) - self.x.queue.delete_message.assert_called_with( - self.x, self.callback, - ) - - self.x.queue = None - assert self.x.delete(callback=self.callback) is None - - def test_change_visibility(self): - assert self.x.change_visibility(303, callback=self.callback) - self.x.queue.connection.change_message_visibility.assert_called_with( - self.x.queue, self.x.receipt_handle, 303, self.callback, - ) - self.x.queue = None - assert self.x.change_visibility(303, callback=self.callback) is None diff --git a/t/unit/transport/test_SQS.py b/t/unit/transport/test_SQS.py index 971f8a55f..ea5f2b5e8 100644 --- a/t/unit/transport/test_SQS.py +++ b/t/unit/transport/test_SQS.py @@ -8,13 +8,14 @@ from __future__ import absolute_import, unicode_literals import pytest +import random +import string from case import Mock, skip from kombu import messaging from kombu import Connection, Exchange, Queue -from kombu.async.aws.ext import exception from kombu.five import Empty from kombu.transport import SQS @@ -28,75 +29,81 @@ def __init__(self): self.receipt_handle = "receipt_handle_xyz" -class SQSQueueMock(object): - """ - Imitate the SQS Queue in boto3. It has two attributes, url and - "attributes". - """ +class QueueMock(object): + """ Hold information about a queue. """ def __init__(self, url): self.url = url self.attributes = {'ApproximateNumberOfMessages': '0'} self.messages = [] - self._receive_messages_calls = 0 - - def send_message(self, MessageBody=None, DelaySeconds=0, MessageAttributes=None): - msg = SQSMessageMock() - msg.body = MessageBody - self.messages.append(msg) - self.attributes['ApproximateNumberOfMessages'] = len(self.messages) - - def receive_messages(self, MaxNumberOfMessages=10): - self._receive_messages_calls += 1 - returning = self.messages[:MaxNumberOfMessages] - self.messages = self.messages[MaxNumberOfMessages:] - self.attributes['ApproximateNumberOfMessages'] = len(self.messages) - return returning - def purge(self): - empty, self.messages[:] = not self.messages, [] - self.attributes['ApproximateNumberOfMessages'] = len(self.messages) - return not empty + def __repr__(self): + return 'QueueMock: {} {} messages'.format(self.url, len(self.messages)) - -class SQSConnectionMock(object): +class SQSClientMock(object): def __init__(self): """ - Imitate the SQS Resource in boto3. - """ - self.queues = self.Queues() - self.queues.queues = { - 'q_%s' % n: SQSQueueMock('q_%s' % n) for n in range(1500) - } - q = SQSQueueMock('unittest_queue') - q.send_message(MessageBody='hello') - self.queues.queues['unittest_queue'] = q - - class Queues: - """ - Imitate the queues attribute on the SQS Resource in boto3. It has - a filter() method and is an iterator rather than a list. + Imitate the SQS Client from boto3. """ - def __init__(self): - self.queues = [] - - def filter(self, QueueNamePrefix=None): - """ Return an iterable of queues. """ - return (val for key, val in self.queues.items() - if key.startswith(QueueNamePrefix)) + self._receive_messages_calls = 0 + # _queues doesn't exist on the real client, here for testing. + self._queues = {} + for n in range(1): + name = 'q_{}'.format(n) + url = 'sqs://q_{}'.format(n) + self.create_queue(QueueName=name) + + url = self.create_queue(QueueName='unittest_queue')['QueueUrl'] + self.send_message(QueueUrl=url, MessageBody='hello') + + def _get_q(self, url): + """ Helper method to quickly get a queue. """ + for q in self._queues.values(): + if q.url == url: + return q + raise Exception("Queue url {} not found".format(url)) def create_queue(self, QueueName=None, Attributes=None): - q = self.queues.queues[QueueName] = SQSQueueMock(QueueName) - return q + q = self._queues[QueueName] = QueueMock('sqs://' + QueueName) + return {'QueueUrl': q.url} + + def list_queues(self, QueueNamePrefix=None): + """ Return a list of queue urls """ + urls = (val.url for key, val in self._queues.items() + if key.startswith(QueueNamePrefix)) + return {'QueueUrls': urls} + + def get_queue_url(self, QueueName=None): + return self._queues[QueueName] + + def send_message(self, QueueUrl=None, MessageBody=None): + for q in self._queues.values(): + if q.url == QueueUrl: + q.messages.append({'Body': MessageBody, 'ReceiptHandle': ''.join(random.choice(string.ascii_lowercase) for x in range(10))}) + break - def get_queue_by_name(self, QueueName=None): - return self.queues.queues.get(QueueName) + def receive_message(self, QueueUrl=None, MaxNumberOfMessages=1): + self._receive_messages_calls += 1 + for q in self._queues.values(): + if q.url == QueueUrl: + msgs = q.messages[:MaxNumberOfMessages] + q.messages = q.messages[MaxNumberOfMessages:] + return {'Messages': msgs} + def get_queue_attributes(self, QueueUrl=None, AttributeNames=None): + if 'ApproximateNumberOfMessages' in AttributeNames: + return {'Attributes': {'ApproximateNumberOfMessages': len(self._get_q(QueueUrl).messages)}} -@skip.unless_module('boto') + def purge_queue(self, QueueUrl=None): + for q in self._queues.values(): + if q.url == QueueUrl: + q.messages = [] + + +@skip.unless_module('boto3') class test_Channel: def handleMessageCallback(self, message): @@ -113,7 +120,7 @@ def setup(self): # Mock the sqs() method that returns an SQSConnection object and # instead return an SQSConnectionMock() object. - self.sqs_conn_mock = SQSConnectionMock() + self.sqs_conn_mock = SQSClientMock() def mock_sqs(): return self.sqs_conn_mock @@ -123,9 +130,9 @@ def mock_sqs(): self.exchange = Exchange('test_SQS', type='direct') self.queue = Queue(self.queue_name, self.exchange, self.queue_name) - # Mock up a test SQS Queue with the SQSQueueMock class (and always + # Mock up a test SQS Queue with the QueueMock class (and always # make sure its a clean empty queue) - self.sqs_queue_mock = SQSQueueMock(self.queue_name) + self.sqs_queue_mock = QueueMock('sqs://' + self.queue_name) # Now, create our Connection object with the SQS Transport and store # the connection/channel objects as references for use in these tests. @@ -158,35 +165,10 @@ def test_init(self): """kombu.SQS.Channel instantiates correctly with mocked queues""" assert self.queue_name in self.channel._queue_cache - # TODO: I dropped the old behavior of changing the error message when - # bad auth is provided. Is this stuff important? - # def test_auth_fail(self): - # normal_func = SQS.Channel.sqs.list_queues - # - # def get_all_queues_fail_403(prefix=''): - # # mock auth error - # raise exception.SQSError(403, None, None) - # - # def get_all_queues_fail_not_403(prefix=''): - # # mock non-auth error - # raise exception.SQSError(500, None, None) - # - # try: - # SQS.Channel.sqs.access_key = '1234' - # SQS.Channel.sqs.get_all_queues = get_all_queues_fail_403 - # with pytest.raises(RuntimeError) as excinfo: - # self.channel = self.connection.channel() - # assert 'access_key=1234' in str(excinfo.value) - # SQS.Channel.sqs.get_all_queues = get_all_queues_fail_not_403 - # with pytest.raises(exception.SQSError): - # self.channel = self.connection.channel() - # finally: - # SQS.Channel.sqs.get_all_queues = normal_func - def test_new_queue(self): queue_name = 'new_unittest_queue' self.channel._new_queue(queue_name) - assert queue_name in self.sqs_conn_mock.queues.queues + assert queue_name in self.sqs_conn_mock._queues.keys() # For cleanup purposes, delete the queue and the queue file self.channel._delete(queue_name) @@ -195,11 +177,13 @@ def test_dont_create_duplicate_new_queue(self): # which is definitely out of cache when get_all_queues returns the # first 1000 queues sorted by name. queue_name = 'unittest_queue' + # This should not create a new queue. self.channel._new_queue(queue_name) - assert queue_name in self.sqs_conn_mock.queues.queues - queue = self.sqs_conn_mock.queues.queues[queue_name] - assert 1 == int(queue.attributes['ApproximateNumberOfMessages']) - assert 'hello' == queue.messages[0].body + assert queue_name in self.sqs_conn_mock._queues.keys() + queue = self.sqs_conn_mock._queues[queue_name] + # The queue originally had 1 message in it. + assert 1 == len(queue.messages) + assert 'hello' == queue.messages[0]['Body'] def test_delete(self): queue_name = 'new_unittest_queue' @@ -211,17 +195,16 @@ def test_get_from_sqs(self): # Test getting a single message message = 'my test message' self.producer.publish(message) - q = self.channel._new_queue(self.queue_name) - results = q.receive_messages() - assert len(results) == 1 + result = self.channel._get(self.queue_name) + assert 'body' in result.keys() # Now test getting many messages for i in range(3): message = 'message: {0}'.format(i) self.producer.publish(message) - results = q.receive_messages(MaxNumberOfMessages=3) - assert len(results) == 3 + self.channel._get_bulk(self.queue_name, max_if_unlimited=3) + assert len(self.sqs_conn_mock._queues[self.queue_name].messages) == 0 def test_get_with_empty_list(self): with pytest.raises(Empty): @@ -244,10 +227,17 @@ def test_messages_to_python(self): message = {'foo': 'bar'} self.channel._put(self.producer.routing_key, message) - q = self.channel._new_queue(self.queue_name) + q_url = self.channel._new_queue(self.queue_name) # Get the messages now - kombu_messages = q.receive_messages(MaxNumberOfMessages=kombu_message_count) - json_messages = q.receive_messages(MaxNumberOfMessages=json_message_count) + kombu_messages = [] + from kombu.async.aws.sqs.ext import Message + for m in self.sqs_conn_mock.receive_message(QueueUrl=q_url, MaxNumberOfMessages=kombu_message_count)['Messages']: + m['Body'] = Message().decode(m['Body']) + kombu_messages.append(m) + json_messages = [] + for m in self.sqs_conn_mock.receive_message(QueueUrl=q_url, MaxNumberOfMessages=json_message_count)['Messages']: + m['Body'] = Message().decode(m['Body']) + json_messages.append(m) # Now convert them to payloads kombu_payloads = self.channel._messages_to_python( @@ -378,6 +368,5 @@ def on_message_delivered(message, queue): assert self.channel.connection._deliver.call_count == message_count - # How many times was the SQSConnectionMock get_message method called? - assert (expected_receive_messages_count == - self.channel._queue_cache[self.queue_name]._receive_messages_calls) + # How many times was the SQSConnectionMock receive_message method called? + assert (expected_receive_messages_count == self.sqs_conn_mock._receive_messages_calls) From 1e4b1e3cbd2d07c68c21d70d6b35c503a5a19906 Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Thu, 12 Jan 2017 12:24:40 -0600 Subject: [PATCH 06/44] Clean up imports --- kombu/async/aws/ext.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/kombu/async/aws/ext.py b/kombu/async/aws/ext.py index e645f9736..ef6af1eb1 100644 --- a/kombu/async/aws/ext.py +++ b/kombu/async/aws/ext.py @@ -3,9 +3,6 @@ from __future__ import absolute_import, unicode_literals try: - import boto3 - - # TODO: old.. import boto except ImportError: # pragma: no cover boto = get_regions = ResultSet = RegionInfo = XmlHandler = None @@ -20,21 +17,22 @@ class BotoError(Exception): exception.SQSError = BotoError exception.SQSDecodeError = BotoError else: - from botocore import exceptions - # from boto3 import exceptions - from boto3 import session - - # TODO: old.. from boto import exception from boto.connection import AWSAuthConnection, AWSQueryConnection from boto.handler import XmlHandler from boto.resultset import ResultSet from boto.regioninfo import RegionInfo, get_regions -__all__ = [ - 'exceptions', - # TODO: old.. - 'exception', 'AWSAuthConnection', 'AWSQueryConnection', +try: + import boto3 + from botocore import exceptions + from boto3 import session +except ImportError: + pass + + +__all__ = [ + 'exceptions','exception', 'AWSAuthConnection', 'AWSQueryConnection', 'XmlHandler', 'ResultSet', 'RegionInfo', 'get_regions', ] From 66870de2730bb474b9a707ac1a15a0794a69ba43 Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Fri, 13 Jan 2017 09:19:30 -0600 Subject: [PATCH 07/44] Fix Python 2 support --- kombu/transport/SQS.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/kombu/transport/SQS.py b/kombu/transport/SQS.py index 1df36db88..3da85f05e 100644 --- a/kombu/transport/SQS.py +++ b/kombu/transport/SQS.py @@ -39,17 +39,18 @@ import socket import string -from urllib.parse import urlparse, urlunparse import uuid +try: + from urllib.parse import urlparse, urlunparse +except ImportError: + from urlparse import urlparse, urlunparse + from vine import transform, ensure_promise, promise from kombu.async import get_event_loop from kombu.async.aws import sqs as _asynsqs -from kombu.async.aws.ext import boto3, exceptions - -# TODO: old.. -from kombu.async.aws.ext import boto +from kombu.async.aws.ext import boto, boto3, exceptions from kombu.async.aws.sqs.connection import AsyncSQSConnection from kombu.async.aws.sqs.message import Message from kombu.five import Empty, range, string_t, text_t From 5f20c80603ed6ce273fe641f28f1cfa64a57b0d5 Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Fri, 20 Jan 2017 10:11:24 -0600 Subject: [PATCH 08/44] Fix receive_message tests --- t/unit/async/aws/sqs/test_connection.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/t/unit/async/aws/sqs/test_connection.py b/t/unit/async/aws/sqs/test_connection.py index ecc22623a..68dfc2f44 100644 --- a/t/unit/async/aws/sqs/test_connection.py +++ b/t/unit/async/aws/sqs/test_connection.py @@ -93,8 +93,9 @@ def test_receive_message(self): self.x.receive_message(queue, 4, callback=self.callback) self.x.get_list.assert_called_with( 'ReceiveMessage', {'MaxNumberOfMessages': 4}, - [('Message', queue.message_class)], - queue.id, callback=self.callback, + [('Message', AsyncMessage)], + queue, callback=self.callback, + parent=queue, ) def test_receive_message__with_visibility_timeout(self): @@ -105,8 +106,9 @@ def test_receive_message__with_visibility_timeout(self): 'MaxNumberOfMessages': 4, 'VisibilityTimeout': 3666, }, - [('Message', queue.message_class)], - queue.id, callback=self.callback, + [('Message', AsyncMessage)], + queue, callback=self.callback, + parent=queue, ) def test_receive_message__with_wait_time_seconds(self): @@ -119,8 +121,9 @@ def test_receive_message__with_wait_time_seconds(self): 'MaxNumberOfMessages': 4, 'WaitTimeSeconds': 303, }, - [('Message', queue.message_class)], - queue.id, callback=self.callback, + [('Message', AsyncMessage)], + queue, callback=self.callback, + parent=queue, ) def test_receive_message__with_attributes(self): @@ -134,8 +137,9 @@ def test_receive_message__with_attributes(self): 'AttributeName.2': 'bar', 'MaxNumberOfMessages': 4, }, - [('Message', queue.message_class)], - queue.id, callback=self.callback, + [('Message', AsyncMessage)], + queue, callback=self.callback, + parent=queue, ) def MockMessage(self, id=None, receipt_handle=None, body=None): @@ -157,10 +161,10 @@ def _set_body(value): def test_delete_message(self): queue = Mock(name='queue') message = self.MockMessage() - self.x.delete_message(queue, message, callback=self.callback) + self.x.delete_message(queue, message.receipt_handle, callback=self.callback) self.x.get_status.assert_called_with( 'DeleteMessage', {'ReceiptHandle': message.receipt_handle}, - queue.id, callback=self.callback, + queue, callback=self.callback, ) def test_delete_message_batch(self): From 8a6fb0edd6a187405880d77d5b8a50e28b82d7e4 Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Mon, 23 Jan 2017 13:53:48 -0600 Subject: [PATCH 09/44] Reformat docstring --- kombu/async/aws/sqs/message.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kombu/async/aws/sqs/message.py b/kombu/async/aws/sqs/message.py index ed4a9060a..52493f4f8 100644 --- a/kombu/async/aws/sqs/message.py +++ b/kombu/async/aws/sqs/message.py @@ -24,9 +24,7 @@ class AsyncMessage(BaseAsyncMessage, Message): """Serialized message.""" def __getitem__(self, item): - """ - Support Boto3-style access on a message. - """ + """Support Boto3-style access on a message.""" if item == 'ReceiptHandle': return self.receipt_handle elif item == 'Body': From bc8e108bb628075d5a12e027ab473016df84da66 Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Mon, 23 Jan 2017 14:02:41 -0600 Subject: [PATCH 10/44] boto3 import changes for CI --- kombu/async/aws/ext.py | 2 +- kombu/async/aws/sqs/ext.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/kombu/async/aws/ext.py b/kombu/async/aws/ext.py index ef6af1eb1..282ddb926 100644 --- a/kombu/async/aws/ext.py +++ b/kombu/async/aws/ext.py @@ -29,7 +29,7 @@ class BotoError(Exception): from botocore import exceptions from boto3 import session except ImportError: - pass + boto3 = exceptions = session = None __all__ = [ diff --git a/kombu/async/aws/sqs/ext.py b/kombu/async/aws/sqs/ext.py index 064155390..7719d4c58 100644 --- a/kombu/async/aws/sqs/ext.py +++ b/kombu/async/aws/sqs/ext.py @@ -4,9 +4,6 @@ from __future__ import absolute_import, unicode_literals try: - import boto3 - - # TODO: old.. import boto except ImportError: # pragma: no cover boto = Attributes = BatchResults = None # noqa @@ -28,6 +25,13 @@ class _void(object): from boto.sqs.connection import SQSConnection from boto.sqs.queue import Queue + +try: + import boto3 +except ImportError: + boto3 = None + + __all__ = [ 'Attributes', 'BatchResults', 'EncodedMHMessage', 'MHMessage', 'Message', 'RawMessage', 'JSONMessage', 'SQSConnection', From 759e09ed9cff7c450c1109617d7efc35892e2885 Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Mon, 23 Jan 2017 14:16:15 -0600 Subject: [PATCH 11/44] skip tests if boto3 not installed --- kombu/async/aws/ext.py | 10 +++++++++- t/integration/tests/test_SQS.py | 1 + t/unit/async/aws/case.py | 1 + t/unit/transport/test_SQS.py | 1 - 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/kombu/async/aws/ext.py b/kombu/async/aws/ext.py index 282ddb926..0ec35e6a9 100644 --- a/kombu/async/aws/ext.py +++ b/kombu/async/aws/ext.py @@ -29,7 +29,15 @@ class BotoError(Exception): from botocore import exceptions from boto3 import session except ImportError: - boto3 = exceptions = session = None + boto3 = session = None + + class _void(object): + pass + + class BotoCoreError(Exception): + pass + exceptions = _void() + exceptions.BotoCoreError = BotoCoreError __all__ = [ diff --git a/t/integration/tests/test_SQS.py b/t/integration/tests/test_SQS.py index 676ba9948..1c52d6489 100644 --- a/t/integration/tests/test_SQS.py +++ b/t/integration/tests/test_SQS.py @@ -8,6 +8,7 @@ @skip.unless_environ('AWS_ACCESS_KEY_ID') @skip.unless_environ('AWS_SECRET_ACCESS_KEY') @skip.unless_module('boto') +@skip.unless_module('boto3') class test_SQS(transport.TransportCase): transport = 'SQS' prefix = 'sqs' diff --git a/t/unit/async/aws/case.py b/t/unit/async/aws/case.py index 985dbc048..9bfce209a 100644 --- a/t/unit/async/aws/case.py +++ b/t/unit/async/aws/case.py @@ -8,6 +8,7 @@ @skip.if_pypy() @skip.unless_module('boto') +@skip.unless_module('boto3') @skip.unless_module('pycurl') @pytest.mark.usefixtures('hub') class AWSCase(object): diff --git a/t/unit/transport/test_SQS.py b/t/unit/transport/test_SQS.py index ea5f2b5e8..b1dad8232 100644 --- a/t/unit/transport/test_SQS.py +++ b/t/unit/transport/test_SQS.py @@ -103,7 +103,6 @@ def purge_queue(self, QueueUrl=None): q.messages = [] -@skip.unless_module('boto3') class test_Channel: def handleMessageCallback(self, message): From c4b55645f42f70be0443f281d39ad04817f51dfb Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Mon, 23 Jan 2017 14:36:40 -0600 Subject: [PATCH 12/44] skip tests if boto3 not installed --- kombu/transport/SQS.py | 2 ++ t/unit/transport/test_SQS.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/kombu/transport/SQS.py b/kombu/transport/SQS.py index 3da85f05e..23ccaf5f4 100644 --- a/kombu/transport/SQS.py +++ b/kombu/transport/SQS.py @@ -98,6 +98,8 @@ class Channel(virtual.Channel): def __init__(self, *args, **kwargs): if boto is None: raise ImportError('boto is not installed') + if boto3 is None: + raise ImportError('boto3 is not installed') super(Channel, self).__init__(*args, **kwargs) # SQS blows up if you try to create a new queue when one already diff --git a/t/unit/transport/test_SQS.py b/t/unit/transport/test_SQS.py index b1dad8232..5feae5263 100644 --- a/t/unit/transport/test_SQS.py +++ b/t/unit/transport/test_SQS.py @@ -103,6 +103,8 @@ def purge_queue(self, QueueUrl=None): q.messages = [] +@skip.unless_module('boto') +@skip.unless_module('boto3') class test_Channel: def handleMessageCallback(self, message): From 90c23b0cd2dc42313271eec594c51eacce87c76c Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Mon, 23 Jan 2017 14:59:46 -0600 Subject: [PATCH 13/44] flake8 --- kombu/async/aws/ext.py | 2 +- kombu/transport/SQS.py | 25 +++++++++++++++++-------- t/unit/async/aws/sqs/test_connection.py | 3 ++- t/unit/transport/test_SQS.py | 24 +++++++++++++++++------- 4 files changed, 37 insertions(+), 17 deletions(-) diff --git a/kombu/async/aws/ext.py b/kombu/async/aws/ext.py index 0ec35e6a9..031232ca4 100644 --- a/kombu/async/aws/ext.py +++ b/kombu/async/aws/ext.py @@ -41,6 +41,6 @@ class BotoCoreError(Exception): __all__ = [ - 'exceptions','exception', 'AWSAuthConnection', 'AWSQueryConnection', + 'exception', 'exceptions', 'AWSAuthConnection', 'AWSQueryConnection', 'XmlHandler', 'ResultSet', 'RegionInfo', 'get_regions', ] diff --git a/kombu/transport/SQS.py b/kombu/transport/SQS.py index 23ccaf5f4..de5a8d40b 100644 --- a/kombu/transport/SQS.py +++ b/kombu/transport/SQS.py @@ -200,14 +200,17 @@ def _delete(self, queue, *args, **kwargs): def _put(self, queue, message, **kwargs): """Put message onto queue.""" q_url = self._new_queue(queue) - kwargs = {'QueueUrl': q_url, 'MessageBody': Message().encode(dumps(message))} + kwargs = {'QueueUrl': q_url, + 'MessageBody': Message().encode(dumps(message))} if queue.endswith('.fifo'): if 'MessageGroupId' in message['properties']: - kwargs['MessageGroupId'] = message['properties']['MessageGroupId'] + kwargs['MessageGroupId'] = \ + message['properties']['MessageGroupId'] else: kwargs['MessageGroupId'] = 'default' if 'MessageDeduplicationId' in message['properties']: - kwargs['MessageDeduplicationId'] = message['properties']['MessageDeduplicationId'] + kwargs['MessageDeduplicationId'] = \ + message['properties']['MessageDeduplicationId'] else: kwargs['MessageDeduplicationId'] = str(uuid.uuid4()) self.sqs.send_message(**kwargs) @@ -283,7 +286,8 @@ def _get_bulk(self, queue, max_count = self._get_message_estimate() if max_count: q_url = self._new_queue(queue) - resp = self.sqs.receive_message(QueueUrl=q_url, MaxNumberOfMessages=max_count) + resp = self.sqs.receive_message( + QueueUrl=q_url, MaxNumberOfMessages=max_count) if resp['Messages']: for m in resp['Messages']: @@ -299,7 +303,8 @@ def _get(self, queue): resp = self.sqs.receive_message(q_url) if resp['Messages']: - resp['Messages'][0]['Body'] = Message().decode(resp['Messages'][0]['Body']) + body = Message().decode(resp['Messages'][0]['Body']) + resp['Messages'][0]['Body'] = body return self._messages_to_python(resp['Messages'], queue)[0] raise Empty() @@ -376,13 +381,16 @@ def basic_ack(self, delivery_tag, multiple=False): except KeyError: pass else: - resp = self.asynsqs.delete_message(message['queue'], message['ReceiptHandle']) + self.asynsqs.delete_message(message['queue'], + message['ReceiptHandle']) super(Channel, self).basic_ack(delivery_tag) def _size(self, queue): """Return the number of messages in a queue.""" url = self._new_queue(queue) - resp = self.sqs.get_queue_attributes(QueueUrl=url, AttributeNames=['ApproximateNumberOfMessages']) + resp = self.sqs.get_queue_attributes( + QueueUrl=url, + AttributeNames=['ApproximateNumberOfMessages']) return int(resp['Attributes']['ApproximateNumberOfMessages']) def _purge(self, queue): @@ -431,11 +439,12 @@ def sqs(self): def asynsqs(self): if self._asynsqs is None: region = self._get_regioninfo(_asynsqs.regions()) + is_secure = self.is_secure if self.is_secure is not None else True self._asynsqs = AsyncSQSConnection( aws_access_key_id=self.conninfo.userid, aws_secret_access_key=self.conninfo.password, region=region, - is_secure=self.is_secure if self.is_secure is not None else True, + is_secure=is_secure, ) return self._asynsqs diff --git a/t/unit/async/aws/sqs/test_connection.py b/t/unit/async/aws/sqs/test_connection.py index 68dfc2f44..9f1866cfc 100644 --- a/t/unit/async/aws/sqs/test_connection.py +++ b/t/unit/async/aws/sqs/test_connection.py @@ -161,7 +161,8 @@ def _set_body(value): def test_delete_message(self): queue = Mock(name='queue') message = self.MockMessage() - self.x.delete_message(queue, message.receipt_handle, callback=self.callback) + self.x.delete_message(queue, message.receipt_handle, + callback=self.callback) self.x.get_status.assert_called_with( 'DeleteMessage', {'ReceiptHandle': message.receipt_handle}, queue, callback=self.callback, diff --git a/t/unit/transport/test_SQS.py b/t/unit/transport/test_SQS.py index 5feae5263..180b33bf3 100644 --- a/t/unit/transport/test_SQS.py +++ b/t/unit/transport/test_SQS.py @@ -73,7 +73,7 @@ def create_queue(self, QueueName=None, Attributes=None): def list_queues(self, QueueNamePrefix=None): """ Return a list of queue urls """ urls = (val.url for key, val in self._queues.items() - if key.startswith(QueueNamePrefix)) + if key.startswith(QueueNamePrefix)) return {'QueueUrls': urls} def get_queue_url(self, QueueName=None): @@ -82,7 +82,10 @@ def get_queue_url(self, QueueName=None): def send_message(self, QueueUrl=None, MessageBody=None): for q in self._queues.values(): if q.url == QueueUrl: - q.messages.append({'Body': MessageBody, 'ReceiptHandle': ''.join(random.choice(string.ascii_lowercase) for x in range(10))}) + handle = ''.join(random.choice(string.ascii_lowercase) for + x in range(10)) + q.messages.append({'Body': MessageBody, + 'ReceiptHandle': handle}) break def receive_message(self, QueueUrl=None, MaxNumberOfMessages=1): @@ -95,7 +98,8 @@ def receive_message(self, QueueUrl=None, MaxNumberOfMessages=1): def get_queue_attributes(self, QueueUrl=None, AttributeNames=None): if 'ApproximateNumberOfMessages' in AttributeNames: - return {'Attributes': {'ApproximateNumberOfMessages': len(self._get_q(QueueUrl).messages)}} + count = len(self._get_q(QueueUrl).messages) + return {'Attributes': {'ApproximateNumberOfMessages': count}} def purge_queue(self, QueueUrl=None): for q in self._queues.values(): @@ -232,11 +236,15 @@ def test_messages_to_python(self): # Get the messages now kombu_messages = [] from kombu.async.aws.sqs.ext import Message - for m in self.sqs_conn_mock.receive_message(QueueUrl=q_url, MaxNumberOfMessages=kombu_message_count)['Messages']: + for m in self.sqs_conn_mock.receive_message( + QueueUrl=q_url, + MaxNumberOfMessages=kombu_message_count)['Messages']: m['Body'] = Message().decode(m['Body']) kombu_messages.append(m) json_messages = [] - for m in self.sqs_conn_mock.receive_message(QueueUrl=q_url, MaxNumberOfMessages=json_message_count)['Messages']: + for m in self.sqs_conn_mock.receive_message( + QueueUrl=q_url, + MaxNumberOfMessages=json_message_count)['Messages']: m['Body'] = Message().decode(m['Body']) json_messages.append(m) @@ -369,5 +377,7 @@ def on_message_delivered(message, queue): assert self.channel.connection._deliver.call_count == message_count - # How many times was the SQSConnectionMock receive_message method called? - assert (expected_receive_messages_count == self.sqs_conn_mock._receive_messages_calls) + # How many times was the SQSConnectionMock receive_message method + # called? + assert (expected_receive_messages_count == + self.sqs_conn_mock._receive_messages_calls) From 2d492a47b2af3e70ac1681c34912f1bac2247858 Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Tue, 14 Feb 2017 17:59:07 -0800 Subject: [PATCH 14/44] noboto --- kombu/async/aws/ext.py | 6 +++--- kombu/async/aws/sqs/__init__.py | 22 ---------------------- kombu/async/aws/sqs/connection.py | 10 ++-------- kombu/transport/SQS.py | 15 +++------------ 4 files changed, 8 insertions(+), 45 deletions(-) diff --git a/kombu/async/aws/ext.py b/kombu/async/aws/ext.py index 031232ca4..26439558d 100644 --- a/kombu/async/aws/ext.py +++ b/kombu/async/aws/ext.py @@ -5,7 +5,7 @@ try: import boto except ImportError: # pragma: no cover - boto = get_regions = ResultSet = RegionInfo = XmlHandler = None + boto = ResultSet = RegionInfo = XmlHandler = None class _void(object): pass @@ -21,7 +21,7 @@ class BotoError(Exception): from boto.connection import AWSAuthConnection, AWSQueryConnection from boto.handler import XmlHandler from boto.resultset import ResultSet - from boto.regioninfo import RegionInfo, get_regions + from boto.regioninfo import RegionInfo try: @@ -42,5 +42,5 @@ class BotoCoreError(Exception): __all__ = [ 'exception', 'exceptions', 'AWSAuthConnection', 'AWSQueryConnection', - 'XmlHandler', 'ResultSet', 'RegionInfo', 'get_regions', + 'XmlHandler', 'ResultSet', 'RegionInfo', ] diff --git a/kombu/async/aws/sqs/__init__.py b/kombu/async/aws/sqs/__init__.py index fe529584c..e69de29bb 100644 --- a/kombu/async/aws/sqs/__init__.py +++ b/kombu/async/aws/sqs/__init__.py @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import, unicode_literals - -from kombu.async.aws.ext import boto, get_regions - -from .connection import AsyncSQSConnection - -__all__ = ['regions', 'connect_to_region'] - - -def regions(): - """Return list of known AWS regions.""" - if boto is None: - raise ImportError('boto is not installed') - return get_regions('sqs', connection_cls=AsyncSQSConnection) - - -def connect_to_region(region_name, **kwargs): - """Connect to specific AWS region.""" - for region in regions(): - if region.name == region_name: - return region.connect(**kwargs) diff --git a/kombu/async/aws/sqs/connection.py b/kombu/async/aws/sqs/connection.py index d73e7ea62..c4a27b19f 100644 --- a/kombu/async/aws/sqs/connection.py +++ b/kombu/async/aws/sqs/connection.py @@ -7,7 +7,7 @@ from kombu.async.aws.connection import AsyncAWSQueryConnection from kombu.async.aws.ext import RegionInfo -from .ext import boto, boto3, Attributes, BatchResults, SQSConnection +from .ext import boto3, Attributes, BatchResults, SQSConnection from .message import AsyncMessage from .queue import AsyncQueue @@ -22,14 +22,8 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, region=None, *args, **kwargs): - if boto is None: - raise ImportError('boto is not installed') if boto3 is None: raise ImportError('boto3 is not installed') - self.region = region or RegionInfo( - self, self.DefaultRegionName, self.DefaultRegionEndpoint, - connection_cls=type(self), - ) AsyncAWSQueryConnection.__init__( self, aws_access_key_id=aws_access_key_id, @@ -37,7 +31,7 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, is_secure=is_secure, port=port, proxy=proxy, proxy_port=proxy_port, proxy_user=proxy_user, proxy_pass=proxy_pass, - host=self.region.endpoint, debug=debug, + region_name=region, debug=debug, https_connection_factory=https_connection_factory, **kwargs ) diff --git a/kombu/transport/SQS.py b/kombu/transport/SQS.py index de5a8d40b..62142af3b 100644 --- a/kombu/transport/SQS.py +++ b/kombu/transport/SQS.py @@ -96,8 +96,8 @@ class Channel(virtual.Channel): _noack_queues = set() def __init__(self, *args, **kwargs): - if boto is None: - raise ImportError('boto is not installed') + # if boto is None: + # raise ImportError('boto is not installed') if boto3 is None: raise ImportError('boto3 is not installed') super(Channel, self).__init__(*args, **kwargs) @@ -415,14 +415,6 @@ def close(self): if "can't set attribute" not in str(exc): raise - def _get_regioninfo(self, regions): - if self.regioninfo: - return self.regioninfo - if self.region: - for _r in regions: - if _r.name == self.region: - return _r - @property def sqs(self): if self._sqs is None: @@ -438,12 +430,11 @@ def sqs(self): @property def asynsqs(self): if self._asynsqs is None: - region = self._get_regioninfo(_asynsqs.regions()) is_secure = self.is_secure if self.is_secure is not None else True self._asynsqs = AsyncSQSConnection( aws_access_key_id=self.conninfo.userid, aws_secret_access_key=self.conninfo.password, - region=region, + region=self.region, is_secure=is_secure, ) return self._asynsqs From 57ea7f1f01dbf0104d211c2f00f6e2095c17d802 Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Tue, 14 Feb 2017 22:59:07 -0800 Subject: [PATCH 15/44] ditching boto2. got queue URL fetching, async HTTP request generation and signing working. --- kombu/async/aws/connection.py | 102 ++++++++++-------------------- kombu/async/aws/ext.py | 40 ++++++------ kombu/async/aws/sqs/connection.py | 24 +++++-- kombu/transport/SQS.py | 36 +++++------ 4 files changed, 88 insertions(+), 114 deletions(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index cc907d7b8..14519460d 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -6,12 +6,14 @@ from vine import promise, transform +from botocore.awsrequest import AWSRequest + from kombu.async.http import Headers, Request, get_client from kombu.five import items, python_2_unicode_compatible -from .ext import ( - boto, AWSAuthConnection, AWSQueryConnection, XmlHandler, ResultSet, -) +# from .ext import ( +# XmlHandler, ResultSet, +# ) try: from urllib.parse import urlunsplit @@ -31,7 +33,6 @@ def message_from_file(m): # noqa __all__ = [ 'AsyncHTTPConnection', 'AsyncHTTPSConnection', 'AsyncHTTPResponse', 'AsyncConnection', - 'AsyncAWSAuthConnection', 'AsyncAWSQueryConnection', ] @@ -78,7 +79,7 @@ def __repr__(self): @python_2_unicode_compatible -class AsyncHTTPConnection(object): +class AsyncHTTPSConnection(object): """Async HTTP Connection.""" Request = Request @@ -87,13 +88,10 @@ class AsyncHTTPConnection(object): method = 'GET' path = '/' body = None - scheme = 'http' + scheme = 'https' default_ports = {'http': 80, 'https': 443} - def __init__(self, host, port=None, - strict=None, timeout=20.0, http_client=None, **kwargs): - self.host = host - self.port = port + def __init__(self, strict=None, timeout=20.0, http_client=None, **kwargs): self.headers = [] self.timeout = timeout self.strict = strict @@ -113,13 +111,8 @@ def request(self, method, path, body=None, headers=None): self.headers.extend(list(items(headers))) def getrequest(self, scheme=None): - scheme = scheme if scheme else self.scheme - host = self.host - if self.port and self.port != self.default_ports[scheme]: - host = '{0}:{1}'.format(host, self.port) - url = urlunsplit((scheme, host, self.path, '', '')) headers = Headers(self.headers) - return self.Request(url, method=self.method, headers=headers, + return self.Request(self.path, method=self.method, headers=headers, body=self.body, connect_timeout=self.timeout, request_timeout=self.timeout, validate_cert=False) @@ -157,85 +150,56 @@ def __repr__(self): return ''.format(self.getrequest()) -class AsyncHTTPSConnection(AsyncHTTPConnection): - """Async HTTPS Connection.""" - - scheme = 'https' - - class AsyncConnection(object): """Async AWS Connection.""" - def __init__(self, http_client=None, **kwargs): - if boto is None: - raise ImportError('boto is not installed') + def __init__(self, sqs_connection, http_client=None, **kwargs): + self.sqs_connection = sqs_connection self._httpclient = http_client or get_client() - def get_http_connection(self, host, port, is_secure): - return (AsyncHTTPSConnection if is_secure else AsyncHTTPConnection)( - host, port, http_client=self._httpclient, - ) + def get_http_connection(self): + return AsyncHTTPSConnection(http_client=self._httpclient) def _mexe(self, request, sender=None, callback=None): callback = callback or promise() - boto.log.debug( - 'HTTP %s/%s headers=%s body=%s', - request.host, request.path, + print( + 'HTTP %s headers=%s body=%s', + request.url, request.headers, request.body, ) - conn = self.get_http_connection( - request.host, request.port, self.is_secure, - ) - request.authorize(connection=self) + conn = self.get_http_connection() if callable(sender): sender(conn, request.method, request.path, request.body, request.headers, callback) else: - conn.request(request.method, request.path, + conn.request(request.method, request.url, request.body, request.headers) conn.getresponse(callback=callback) return callback -class AsyncAWSAuthConnection(AsyncConnection, AWSAuthConnection): - """Async AWS Authn Connection.""" - - def __init__(self, host, - http_client=None, http_client_params={}, **kwargs): - AsyncConnection.__init__(self, http_client, **http_client_params) - AWSAuthConnection.__init__(self, host, **kwargs) - - def make_request(self, method, path, headers=None, data='', host=None, - auth_path=None, sender=None, callback=None, **kwargs): - req = self.build_base_http_request( - method, path, auth_path, {}, headers, data, host, - ) - return self._mexe(req, sender=sender, callback=callback) - - -class AsyncAWSQueryConnection(AsyncConnection, AWSQueryConnection): +class AsyncAWSQueryConnection(AsyncConnection): """Async AWS Query Connection.""" - def __init__(self, host, - http_client=None, http_client_params={}, **kwargs): - AsyncConnection.__init__(self, http_client, **http_client_params) - AWSAuthConnection.__init__(self, host, **kwargs) + def __init__(self, sqs_connection, http_client=None, http_client_params={}, **kwargs): + AsyncConnection.__init__(self, sqs_connection, http_client, **http_client_params) - def make_request(self, action, params, path, verb, callback=None): - request = self.build_base_http_request( - verb, path, None, params, {}, '', self.server_name()) + def make_request(self, action, params_, path, verb, callback=None): + params = params_.copy() if action: - request.params['Action'] = action - request.params['Version'] = self.APIVersion - return self._mexe(request, callback=callback) + params['Action'] = action + signer = self.sqs_connection._request_signer + request = AWSRequest(method=verb, url=path, params=params) + signer.sign(action, request) + return self._mexe(request.prepare(), callback=callback) def get_list(self, action, params, markers, path='/', parent=None, verb='GET', callback=None): return self.make_request( action, params, path, verb, - callback=transform( + callback=transform( self._on_list_ready, callback, parent or self, markers, ), ) @@ -289,7 +253,7 @@ def _on_status_ready(self, parent, response): raise self._for_status(response, body) def _for_status(self, response, body): - context = 'Empty body' if not body else 'HTTP Error' - exc = self.ResponseError(response.status, response.reason, body) - boto.log.error('{0}: %r'.format(context), exc) - return exc + return Exception("\n".join([str(response), str(body)])) + # context = 'Empty body' if not body else 'HTTP Error' + # exc = self.ResponseError(response.status, response.reason, body) + # return exc diff --git a/kombu/async/aws/ext.py b/kombu/async/aws/ext.py index 26439558d..f871d63b8 100644 --- a/kombu/async/aws/ext.py +++ b/kombu/async/aws/ext.py @@ -2,26 +2,26 @@ """Amazon boto3 interface.""" from __future__ import absolute_import, unicode_literals -try: - import boto -except ImportError: # pragma: no cover - boto = ResultSet = RegionInfo = XmlHandler = None - - class _void(object): - pass - AWSAuthConnection = AWSQueryConnection = _void # noqa - - class BotoError(Exception): - pass - exception = _void() - exception.SQSError = BotoError - exception.SQSDecodeError = BotoError -else: - from boto import exception - from boto.connection import AWSAuthConnection, AWSQueryConnection - from boto.handler import XmlHandler - from boto.resultset import ResultSet - from boto.regioninfo import RegionInfo +# try: +# import boto +# except ImportError: # pragma: no cover +# boto = ResultSet = RegionInfo = XmlHandler = None + +# class _void(object): +# pass +# AWSAuthConnection = AWSQueryConnection = _void # noqa + +# class BotoError(Exception): +# pass +# exception = _void() +# exception.SQSError = BotoError +# exception.SQSDecodeError = BotoError +# else: +# from boto import exception +# from boto.connection import AWSAuthConnection, AWSQueryConnection +# from boto.handler import XmlHandler +# from boto.resultset import ResultSet +# from boto.regioninfo import RegionInfo try: diff --git a/kombu/async/aws/sqs/connection.py b/kombu/async/aws/sqs/connection.py index c4a27b19f..c1efa3651 100644 --- a/kombu/async/aws/sqs/connection.py +++ b/kombu/async/aws/sqs/connection.py @@ -5,11 +5,12 @@ from vine import transform from kombu.async.aws.connection import AsyncAWSQueryConnection -from kombu.async.aws.ext import RegionInfo from .ext import boto3, Attributes, BatchResults, SQSConnection from .message import AsyncMessage from .queue import AsyncQueue +from botocore.exceptions import ClientError + __all__ = ['AsyncSQSConnection'] @@ -18,17 +19,17 @@ class AsyncSQSConnection(AsyncAWSQueryConnection, SQSConnection): """Async SQS Connection.""" - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, + def __init__(self, sqs_connection, aws_access_key_id=None, aws_secret_access_key=None, + proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, region=None, *args, **kwargs): if boto3 is None: raise ImportError('boto3 is not installed') AsyncAWSQueryConnection.__init__( self, + sqs_connection, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, - is_secure=is_secure, port=port, proxy=proxy, proxy_port=proxy_port, proxy_user=proxy_user, proxy_pass=proxy_pass, region_name=region, debug=debug, @@ -49,6 +50,18 @@ def delete_queue(self, queue, force_deletion=False, callback=None): return self.get_status('DeleteQueue', None, queue.id, callback=callback) + def get_queue_url(self, queue): + res = self.sqs_connection.get_queue_url(QueueName=queue) + # try: + # except ClientError as e: + # print("Failed to get queue URL", e.response['Error']['Code']) + # if e.response['Error']['Code'] == 'AWS.SimpleQueueService.NonExistentQueue': + # return None + # raise e + # print("code", e['Error']['Code']) + # return None + return res['QueueUrl'] + def get_queue_attributes(self, queue, attribute='All', callback=None): return self.get_object( 'GetQueueAttributes', {'AttributeName': attribute}, @@ -73,9 +86,10 @@ def receive_message(self, queue, self.build_list_params(params, attributes, 'AttributeName') if wait_time_seconds is not None: params['WaitTimeSeconds'] = wait_time_seconds + queue_url = self.get_queue_url(queue) return self.get_list( 'ReceiveMessage', params, [('Message', AsyncMessage)], - queue, callback=callback, parent=queue, + queue_url, callback=callback, parent=queue, ) def delete_message(self, queue, receipt_handle, callback=None): diff --git a/kombu/transport/SQS.py b/kombu/transport/SQS.py index 62142af3b..5b784cd90 100644 --- a/kombu/transport/SQS.py +++ b/kombu/transport/SQS.py @@ -41,16 +41,11 @@ import string import uuid -try: - from urllib.parse import urlparse, urlunparse -except ImportError: - from urlparse import urlparse, urlunparse - from vine import transform, ensure_promise, promise from kombu.async import get_event_loop from kombu.async.aws import sqs as _asynsqs -from kombu.async.aws.ext import boto, boto3, exceptions +from kombu.async.aws.ext import boto3, exceptions from kombu.async.aws.sqs.connection import AsyncSQSConnection from kombu.async.aws.sqs.message import Message from kombu.five import Empty, range, string_t, text_t @@ -96,8 +91,6 @@ class Channel(virtual.Channel): _noack_queues = set() def __init__(self, *args, **kwargs): - # if boto is None: - # raise ImportError('boto is not installed') if boto3 is None: raise ImportError('boto3 is not installed') super(Channel, self).__init__(*args, **kwargs) @@ -115,6 +108,7 @@ def _update_queue_cache(self, queue_name_prefix): for url in resp.get('QueueUrls', []): queue_name = url.split('/')[-1] self._queue_cache[queue_name] = url + print(queue_name, url) def basic_consume(self, queue, no_ack, *args, **kwargs): if no_ack: @@ -166,13 +160,16 @@ def entity_name(self, name, table=CHARS_REPLACE_TABLE): else: return text_t(safe_str(name)).translate(table) + def canonical_queue_name(self, queue_name): + return self.entity_name(self.queue_name_prefix + queue_name) + def _new_queue(self, queue, **kwargs): """Ensure a queue with given name exists in SQS.""" if not isinstance(queue, string_t): return queue # Translate to SQS name for consistency with initial # _queue_cache population. - queue = self.entity_name(self.queue_name_prefix + queue) + queue = self.canonical_queue_name(queue) # The SQS ListQueues method only returns 1000 queues. When you have # so many queues, it's possible that the queue you are looking for is @@ -339,12 +336,9 @@ def _get_bulk_async(self, queue, def _get_async(self, queue, count=1, callback=None): q = self._new_queue(queue) - result = list(urlparse(q)) - result[0] = result[1] = '' - q_shortpath = urlunparse(result) - + qname = self.canonical_queue_name(queue) return self._get_from_sqs( - q_shortpath, count=count, connection=self.asynsqs, + qname, count=count, connection=self.asynsqs, callback=transform(self._on_messages_ready, callback, q, queue), ) @@ -362,6 +356,7 @@ def _get_from_sqs(self, queue, Uses long polling and returns :class:`~vine.promises.promise`. """ connection = connection if connection is not None else queue.connection + # url = self.get_queue return connection.receive_message( queue, number_messages=count, wait_time_seconds=self.wait_time_seconds, @@ -408,12 +403,12 @@ def _purge(self, queue): def close(self): super(Channel, self).close() - if self._asynsqs: - try: - self.asynsqs.close() - except AttributeError as exc: # FIXME ??? - if "can't set attribute" not in str(exc): - raise + # if self._asynsqs: + # try: + # self.asynsqs.close() + # except AttributeError as exc: # FIXME ??? + # if "can't set attribute" not in str(exc): + # raise @property def sqs(self): @@ -432,6 +427,7 @@ def asynsqs(self): if self._asynsqs is None: is_secure = self.is_secure if self.is_secure is not None else True self._asynsqs = AsyncSQSConnection( + sqs_connection=self.sqs, aws_access_key_id=self.conninfo.userid, aws_secret_access_key=self.conninfo.password, region=self.region, From fbab8968d37763430924bee7348d87a7011f01bc Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 00:29:53 -0800 Subject: [PATCH 16/44] request signing working kinda --- kombu/async/aws/connection.py | 155 ++++++++++-------------------- kombu/async/aws/ext.py | 25 +---- kombu/async/aws/sqs/connection.py | 10 -- kombu/async/http/curl.py | 2 +- kombu/transport/SQS.py | 1 - 5 files changed, 54 insertions(+), 139 deletions(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index 14519460d..dc2c7fd97 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -2,19 +2,16 @@ """Amazon AWS Connection.""" from __future__ import absolute_import, unicode_literals -from io import BytesIO - from vine import promise, transform +import requests + from botocore.awsrequest import AWSRequest +from botocore.response import get_response from kombu.async.http import Headers, Request, get_client from kombu.five import items, python_2_unicode_compatible -# from .ext import ( -# XmlHandler, ResultSet, -# ) - try: from urllib.parse import urlunsplit except ImportError: @@ -31,59 +28,16 @@ def message_from_file(m): # noqa return m __all__ = [ - 'AsyncHTTPConnection', 'AsyncHTTPSConnection', - 'AsyncHTTPResponse', 'AsyncConnection', + 'AsyncHTTPSConnection', 'AsyncConnection', ] -@python_2_unicode_compatible -class AsyncHTTPResponse(object): - """Async HTTP Response.""" - - def __init__(self, response): - self.response = response - self._msg = None - self.version = 10 - - def read(self, *args, **kwargs): - return self.response.body - - def getheader(self, name, default=None): - return self.response.headers.get(name, default) - - def getheaders(self): - return list(items(self.response.headers)) - - @property - def msg(self): - if self._msg is None: - self._msg = MIMEMessage(message_from_file( - BytesIO(b'\r\n'.join( - b'{0}: {1}'.format(*h) for h in self.getheaders()) - ) - )) - return self._msg - - @property - def status(self): - return self.response.code - - @property - def reason(self): - if self.response.error: - return self.response.error.message - return '' - - def __repr__(self): - return repr(self.response) - - @python_2_unicode_compatible class AsyncHTTPSConnection(object): """Async HTTP Connection.""" Request = Request - Response = AsyncHTTPResponse + Response = requests.Response method = 'GET' path = '/' @@ -162,11 +116,11 @@ def get_http_connection(self): def _mexe(self, request, sender=None, callback=None): callback = callback or promise() - print( - 'HTTP %s headers=%s body=%s', - request.url, - request.headers, request.body, - ) + # print( + # 'HTTP request', + # request.url, + # request.headers, request.body, + # ) conn = self.get_http_connection() @@ -186,74 +140,69 @@ class AsyncAWSQueryConnection(AsyncConnection): def __init__(self, sqs_connection, http_client=None, http_client_params={}, **kwargs): AsyncConnection.__init__(self, sqs_connection, http_client, **http_client_params) - def make_request(self, action, params_, path, verb, callback=None): + def make_request(self, operation, params_, path, verb, callback=None): params = params_.copy() - if action: - params['Action'] = action + if operation: + params['Action'] = operation signer = self.sqs_connection._request_signer - request = AWSRequest(method=verb, url=path, params=params) - signer.sign(action, request) - return self._mexe(request.prepare(), callback=callback) - - def get_list(self, action, params, markers, - path='/', parent=None, verb='GET', callback=None): + request = AWSRequest(method=verb, url=path, data=params) + signing_type = 'presign-url' if verb == 'GET' else 'standard' + signer.sign(operation, request, signing_type=signing_type) + prepared_request = request.prepare() + # print(prepared_request.url) + # print(prepared_request.headers) + # print(prepared_request.body) + return self._mexe(request, callback=callback) + + def get_list(self, operation, params, markers, + path='/', parent=None, verb='POST', callback=None): return self.make_request( - action, params, path, verb, + operation, params, path, verb, callback=transform( - self._on_list_ready, callback, parent or self, markers, + self._on_list_ready, callback, parent or self, markers, operation ), ) - def get_object(self, action, params, cls, + def get_object(self, operation, params, cls, path='/', parent=None, verb='GET', callback=None): return self.make_request( - action, params, path, verb, + operation, params, path, verb, callback=transform( - self._on_obj_ready, callback, parent or self, cls, + self._on_obj_ready, callback, parent or self, cls, operation ), ) - def get_status(self, action, params, + def get_status(self, operation, params, path='/', parent=None, verb='GET', callback=None): return self.make_request( - action, params, path, verb, + operation, params, path, verb, callback=transform( - self._on_status_ready, callback, parent or self, + self._on_status_ready, callback, parent or self, operation ), ) - def _on_list_ready(self, parent, markers, response): - body = response.read() - if response.status == 200 and body: - rs = ResultSet(markers) - h = XmlHandler(rs, parent) - sax_parse(body, h) - return rs + def _on_list_ready(self, parent, markers, operation, response): + service_model = self.sqs_connection.meta.service_model + print("OP", operation) + if response.status == 200: + return get_response(service_model.operation_model(operation), response) else: - raise self._for_status(response, body) - - def _on_obj_ready(self, parent, cls, response): - body = response.read() - if response.status == 200 and body: - obj = cls(parent) - h = XmlHandler(obj, parent) - sax_parse(body, h) - return obj + raise self._for_status(response, response.read()) + + def _on_obj_ready(self, parent, cls, operation, response): + service_model = self.sqs_connection.meta.service_model + if response.status == 200: + return get_response(service_model.operation_model(operation), response) else: - raise self._for_status(response, body) - - def _on_status_ready(self, parent, response): - body = response.read() - if response.status == 200 and body: - rs = ResultSet() - h = XmlHandler(rs, parent) - sax_parse(body, h) - return rs.status + raise self._for_status(response, response.read()) + + def _on_status_ready(self, parent, operation, response): + service_model = self.sqs_connection.meta.service_model + if response.status == 200: + return get_response(service_model.operation_model(operation), response) else: - raise self._for_status(response, body) + raise self._for_status(response, response.read()) def _for_status(self, response, body): - return Exception("\n".join([str(response), str(body)])) - # context = 'Empty body' if not body else 'HTTP Error' - # exc = self.ResponseError(response.status, response.reason, body) - # return exc + context = 'Empty body' if not body else 'HTTP Error' + return Exception("Request {} - HTTP {} - {} ({})".format(context, response.status, response.reason, body)) diff --git a/kombu/async/aws/ext.py b/kombu/async/aws/ext.py index f871d63b8..2c7e7ed3b 100644 --- a/kombu/async/aws/ext.py +++ b/kombu/async/aws/ext.py @@ -2,28 +2,6 @@ """Amazon boto3 interface.""" from __future__ import absolute_import, unicode_literals -# try: -# import boto -# except ImportError: # pragma: no cover -# boto = ResultSet = RegionInfo = XmlHandler = None - -# class _void(object): -# pass -# AWSAuthConnection = AWSQueryConnection = _void # noqa - -# class BotoError(Exception): -# pass -# exception = _void() -# exception.SQSError = BotoError -# exception.SQSDecodeError = BotoError -# else: -# from boto import exception -# from boto.connection import AWSAuthConnection, AWSQueryConnection -# from boto.handler import XmlHandler -# from boto.resultset import ResultSet -# from boto.regioninfo import RegionInfo - - try: import boto3 from botocore import exceptions @@ -41,6 +19,5 @@ class BotoCoreError(Exception): __all__ = [ - 'exception', 'exceptions', 'AWSAuthConnection', 'AWSQueryConnection', - 'XmlHandler', 'ResultSet', 'RegionInfo', + 'exceptions' ] diff --git a/kombu/async/aws/sqs/connection.py b/kombu/async/aws/sqs/connection.py index c1efa3651..200a4b509 100644 --- a/kombu/async/aws/sqs/connection.py +++ b/kombu/async/aws/sqs/connection.py @@ -9,8 +9,6 @@ from .ext import boto3, Attributes, BatchResults, SQSConnection from .message import AsyncMessage from .queue import AsyncQueue -from botocore.exceptions import ClientError - __all__ = ['AsyncSQSConnection'] @@ -52,14 +50,6 @@ def delete_queue(self, queue, force_deletion=False, callback=None): def get_queue_url(self, queue): res = self.sqs_connection.get_queue_url(QueueName=queue) - # try: - # except ClientError as e: - # print("Failed to get queue URL", e.response['Error']['Code']) - # if e.response['Error']['Code'] == 'AWS.SimpleQueueService.NonExistentQueue': - # return None - # raise e - # print("code", e['Error']['Code']) - # return None return res['QueueUrl'] def get_queue_attributes(self, queue, attribute='All', callback=None): diff --git a/kombu/async/http/curl.py b/kombu/async/http/curl.py index 1c50eef8a..85117a585 100644 --- a/kombu/async/http/curl.py +++ b/kombu/async/http/curl.py @@ -243,7 +243,7 @@ def _setup_request(self, curl, request, buffer, headers, _pycurl=pycurl): setopt(meth, True) if request.method in ('POST', 'PUT'): - body = request.body or '' + body = request.body or bytes() reqbuffer = BytesIO(body) setopt(_pycurl.READFUNCTION, reqbuffer.read) if request.method == 'POST': diff --git a/kombu/transport/SQS.py b/kombu/transport/SQS.py index 5b784cd90..41827590c 100644 --- a/kombu/transport/SQS.py +++ b/kombu/transport/SQS.py @@ -108,7 +108,6 @@ def _update_queue_cache(self, queue_name_prefix): for url in resp.get('QueueUrls', []): queue_name = url.split('/')[-1] self._queue_cache[queue_name] = url - print(queue_name, url) def basic_consume(self, queue, no_ack, *args, **kwargs): if no_ack: From f8f568ac11307db49f41fe3251cd1d9822dfce78 Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 01:12:16 -0800 Subject: [PATCH 17/44] async parsing of SQS message response more or less working --- kombu/async/aws/connection.py | 70 +++++++++++++++++++++++++++-------- kombu/async/http/base.py | 9 +++++ kombu/async/http/curl.py | 20 +++++----- kombu/transport/SQS.py | 8 ++-- 4 files changed, 77 insertions(+), 30 deletions(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index dc2c7fd97..aff4eef23 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- +# * coding: utf8 * """Amazon AWS Connection.""" from __future__ import absolute_import, unicode_literals from vine import promise, transform -import requests +from io import BytesIO from botocore.awsrequest import AWSRequest from botocore.response import get_response @@ -32,12 +32,54 @@ def message_from_file(m): # noqa ] +@python_2_unicode_compatible +class AsyncHTTPResponse(object): + """Async HTTP Response.""" + + def __init__(self, response): + self.response = response + self._msg = None + self.version = 10 + + def read(self, *args, **kwargs): + return self.response.body + + def getheader(self, name, default=None): + return self.response.headers.get(name, default) + + def getheaders(self): + return list(items(self.response.headers)) + + @property + def msg(self): + if self._msg is None: + self._msg = MIMEMessage(message_from_file( + BytesIO(b'\r\n'.join( + b'{0}: {1}'.format(*h) for h in self.getheaders()) + ) + )) + return self._msg + + @property + def status(self): + return self.response.code + + @property + def reason(self): + if self.response.error: + return self.response.error.message + return '' + + def __repr__(self): + return repr(self.response) + + @python_2_unicode_compatible class AsyncHTTPSConnection(object): """Async HTTP Connection.""" Request = Request - Response = requests.Response + Response = AsyncHTTPResponse method = 'GET' path = '/' @@ -116,12 +158,6 @@ def get_http_connection(self): def _mexe(self, request, sender=None, callback=None): callback = callback or promise() - # print( - # 'HTTP request', - # request.url, - # request.headers, request.body, - # ) - conn = self.get_http_connection() if callable(sender): @@ -146,13 +182,13 @@ def make_request(self, operation, params_, path, verb, callback=None): params['Action'] = operation signer = self.sqs_connection._request_signer request = AWSRequest(method=verb, url=path, data=params) - signing_type = 'presign-url' if verb == 'GET' else 'standard' + signing_type = 'presignurl' if verb == 'GET' else 'standard' signer.sign(operation, request, signing_type=signing_type) prepared_request = request.prepare() # print(prepared_request.url) # print(prepared_request.headers) # print(prepared_request.body) - return self._mexe(request, callback=callback) + return self._mexe(prepared_request, callback=callback) def get_list(self, operation, params, markers, path='/', parent=None, verb='POST', callback=None): @@ -183,26 +219,28 @@ def get_status(self, operation, params, def _on_list_ready(self, parent, markers, operation, response): service_model = self.sqs_connection.meta.service_model - print("OP", operation) if response.status == 200: - return get_response(service_model.operation_model(operation), response) + httpres, parsed = get_response(service_model.operation_model(operation), response.response) + return parsed else: raise self._for_status(response, response.read()) def _on_obj_ready(self, parent, cls, operation, response): service_model = self.sqs_connection.meta.service_model if response.status == 200: - return get_response(service_model.operation_model(operation), response) + httpres, parsed = get_response(service_model.operation_model(operation), response.response) + return parsed else: raise self._for_status(response, response.read()) def _on_status_ready(self, parent, operation, response): service_model = self.sqs_connection.meta.service_model if response.status == 200: - return get_response(service_model.operation_model(operation), response) + httpres, parsed = get_response(service_model.operation_model(operation), response.response) + return httpres['HTTPStatusCode'] else: raise self._for_status(response, response.read()) def _for_status(self, response, body): context = 'Empty body' if not body else 'HTTP Error' - return Exception("Request {} - HTTP {} - {} ({})".format(context, response.status, response.reason, body)) + return Exception("Request {} HTTP {} {} ({})".format(context, response.status, response.reason, body)) diff --git a/kombu/async/http/base.py b/kombu/async/http/base.py index f8a8bc0a8..42c117dab 100644 --- a/kombu/async/http/base.py +++ b/kombu/async/http/base.py @@ -200,6 +200,15 @@ def body(self): self._body = self.buffer.getvalue() return self._body + # these are for compatibility with Requests + @property + def status_code(self): + return self.code + + @property + def content(self): + return self.body + @coro def header_parser(keyt=normalize_header): diff --git a/kombu/async/http/curl.py b/kombu/async/http/curl.py index 85117a585..e54cd0a1a 100644 --- a/kombu/async/http/curl.py +++ b/kombu/async/http/curl.py @@ -171,15 +171,15 @@ def _process(self, curl, errno=None, reason=None, _pycurl=pycurl): code = curl.getinfo(_pycurl.HTTP_CODE) effective_url = curl.getinfo(_pycurl.EFFECTIVE_URL) buffer.seek(0) - try: - request = info['request'] - request.on_ready(self.Response( - request=request, code=code, headers=info['headers'], - buffer=buffer, effective_url=effective_url, error=error, - )) - except Exception as exc: - self.hub.on_callback_error(request.on_ready, exc) - raise + # try: + request = info['request'] + request.on_ready(self.Response( + request=request, code=code, headers=info['headers'], + buffer=buffer, effective_url=effective_url, error=error, + )) + # except Exception as exc: + # self.hub.on_callback_error(request.on_ready, exc) + # raise def _setup_request(self, curl, request, buffer, headers, _pycurl=pycurl): setopt = curl.setopt @@ -243,7 +243,7 @@ def _setup_request(self, curl, request, buffer, headers, _pycurl=pycurl): setopt(meth, True) if request.method in ('POST', 'PUT'): - body = request.body or bytes() + body = request.body.encode('utf-8') if request.body else bytes() reqbuffer = BytesIO(body) setopt(_pycurl.READFUNCTION, reqbuffer.read) if request.method == 'POST': diff --git a/kombu/transport/SQS.py b/kombu/transport/SQS.py index 41827590c..1ef4dcfcc 100644 --- a/kombu/transport/SQS.py +++ b/kombu/transport/SQS.py @@ -342,11 +342,11 @@ def _get_async(self, queue, count=1, callback=None): ) def _on_messages_ready(self, queue, qname, messages): - if messages: + if 'Messages' in messages and messages['Messages']: callbacks = self.connection._callbacks - for raw_message in messages: - message = self._message_to_python(raw_message, qname, queue) - callbacks[qname](message) + for msg in messages['Messages']: + msg_parsed = self._message_to_python(msg, qname, queue) + callbacks[qname](msg_parsed) def _get_from_sqs(self, queue, count=1, connection=None, callback=None): From 053db8cc0eb5cd89ff4e178782fa934ae6df7eb3 Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 01:20:21 -0800 Subject: [PATCH 18/44] botocore sqs dep --- kombu/async/aws/sqs/ext.py | 30 ------------------------------ requirements/extras/sqs.txt | 2 +- 2 files changed, 1 insertion(+), 31 deletions(-) diff --git a/kombu/async/aws/sqs/ext.py b/kombu/async/aws/sqs/ext.py index 7719d4c58..09fdf1a12 100644 --- a/kombu/async/aws/sqs/ext.py +++ b/kombu/async/aws/sqs/ext.py @@ -3,37 +3,7 @@ from __future__ import absolute_import, unicode_literals -try: - import boto -except ImportError: # pragma: no cover - boto = Attributes = BatchResults = None # noqa - - class _void(object): - pass - regions = SQSConnection = Queue = _void - - RawMessage = Message = MHMessage = \ - EncodedMHMessage = JSONMessage = _void -else: - from boto.sqs.attributes import Attributes - from boto.sqs.batchresults import BatchResults - from boto.sqs.message import ( - EncodedMHMessage, Message, MHMessage, RawMessage, - ) - from boto.sqs import regions - from boto.sqs.jsonmessage import JSONMessage - from boto.sqs.connection import SQSConnection - from boto.sqs.queue import Queue - - try: import boto3 except ImportError: boto3 = None - - -__all__ = [ - 'Attributes', 'BatchResults', 'EncodedMHMessage', 'MHMessage', - 'Message', 'RawMessage', 'JSONMessage', 'SQSConnection', - 'Queue', 'regions', -] diff --git a/requirements/extras/sqs.txt b/requirements/extras/sqs.txt index 21046fbd6..d278da8f6 100644 --- a/requirements/extras/sqs.txt +++ b/requirements/extras/sqs.txt @@ -1,3 +1,3 @@ -boto>=2.8 boto3 +botocore pycurl From bf8148e02c1a4d741e64555c2e466da18782c3df Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 01:35:04 -0800 Subject: [PATCH 19/44] ripping out more old boto2 stuff --- kombu/async/aws/connection.py | 6 +++--- kombu/async/aws/sqs/connection.py | 16 ++++++++-------- kombu/async/aws/sqs/message.py | 25 ++----------------------- kombu/async/aws/sqs/queue.py | 3 +-- kombu/transport/SQS.py | 2 +- 5 files changed, 15 insertions(+), 37 deletions(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index aff4eef23..8e7339c4d 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -199,12 +199,12 @@ def get_list(self, operation, params, markers, ), ) - def get_object(self, operation, params, cls, + def get_object(self, operation, params, path='/', parent=None, verb='GET', callback=None): return self.make_request( operation, params, path, verb, callback=transform( - self._on_obj_ready, callback, parent or self, cls, operation + self._on_obj_ready, callback, parent or self, operation ), ) @@ -225,7 +225,7 @@ def _on_list_ready(self, parent, markers, operation, response): else: raise self._for_status(response, response.read()) - def _on_obj_ready(self, parent, cls, operation, response): + def _on_obj_ready(self, parent, operation, response): service_model = self.sqs_connection.meta.service_model if response.status == 200: httpres, parsed = get_response(service_model.operation_model(operation), response.response) diff --git a/kombu/async/aws/sqs/connection.py b/kombu/async/aws/sqs/connection.py index 200a4b509..32a94af86 100644 --- a/kombu/async/aws/sqs/connection.py +++ b/kombu/async/aws/sqs/connection.py @@ -6,7 +6,7 @@ from kombu.async.aws.connection import AsyncAWSQueryConnection -from .ext import boto3, Attributes, BatchResults, SQSConnection +from .ext import boto3 from .message import AsyncMessage from .queue import AsyncQueue @@ -14,7 +14,7 @@ __all__ = ['AsyncSQSConnection'] -class AsyncSQSConnection(AsyncAWSQueryConnection, SQSConnection): +class AsyncSQSConnection(AsyncAWSQueryConnection): """Async SQS Connection.""" def __init__(self, sqs_connection, aws_access_key_id=None, aws_secret_access_key=None, @@ -41,7 +41,7 @@ def create_queue(self, queue_name, params['DefaultVisibilityTimeout'] = format( visibility_timeout, 'd', ) - return self.get_object('CreateQueue', params, AsyncQueue, + return self.get_object('CreateQueue', params, callback=callback) def delete_queue(self, queue, force_deletion=False, callback=None): @@ -55,7 +55,7 @@ def get_queue_url(self, queue): def get_queue_attributes(self, queue, attribute='All', callback=None): return self.get_object( 'GetQueueAttributes', {'AttributeName': attribute}, - Attributes, queue.id, callback=callback, + queue.id, callback=callback, ) def set_queue_attribute(self, queue, attribute, value, callback=None): @@ -96,7 +96,7 @@ def delete_message_batch(self, queue, messages, callback=None): '{0}.ReceiptHandle'.format(prefix): m.receipt_handle, }) return self.get_object( - 'DeleteMessageBatch', params, BatchResults, queue.id, + 'DeleteMessageBatch', params, queue.id, verb='POST', callback=callback, ) @@ -113,7 +113,7 @@ def send_message(self, queue, message_content, if delay_seconds: params['DelaySeconds'] = int(delay_seconds) return self.get_object( - 'SendMessage', params, AsyncMessage, queue.id, + 'SendMessage', params, queue.id, verb='POST', callback=callback, ) @@ -127,7 +127,7 @@ def send_message_batch(self, queue, messages, callback=None): '{0}.DelaySeconds'.format(prefix): msg[2], }) return self.get_object( - 'SendMessageBatch', params, BatchResults, queue.id, + 'SendMessageBatch', params, queue.id, verb='POST', callback=callback, ) @@ -150,7 +150,7 @@ def change_message_visibility_batch(self, queue, messages, callback=None): '{0}.VisibilityTimeout'.format(pre): t[1], }) return self.get_object( - 'ChangeMessageVisibilityBatch', params, BatchResults, queue.id, + 'ChangeMessageVisibilityBatch', params, queue.id, verb='POST', callback=callback, ) diff --git a/kombu/async/aws/sqs/message.py b/kombu/async/aws/sqs/message.py index 52493f4f8..bd40667ff 100644 --- a/kombu/async/aws/sqs/message.py +++ b/kombu/async/aws/sqs/message.py @@ -2,25 +2,16 @@ """Amazon SQS message implementation.""" from __future__ import absolute_import, unicode_literals -from .ext import ( - RawMessage, Message, MHMessage, EncodedMHMessage, JSONMessage, -) - -__all__ = [ - 'BaseAsyncMessage', 'AsyncRawMessage', 'AsyncMessage', - 'AsyncMHMessage', 'AsyncEncodedMHMessage', 'AsyncJSONMessage', -] - class BaseAsyncMessage(object): """Base class for messages received on async client.""" -class AsyncRawMessage(BaseAsyncMessage, RawMessage): +class AsyncRawMessage(BaseAsyncMessage): """Raw Message.""" -class AsyncMessage(BaseAsyncMessage, Message): +class AsyncMessage(BaseAsyncMessage): """Serialized message.""" def __getitem__(self, item): @@ -33,15 +24,3 @@ def __getitem__(self, item): return self.queue else: raise KeyError(item) - - -class AsyncMHMessage(BaseAsyncMessage, MHMessage): - """MHM Message (uhm, look that up later).""" - - -class AsyncEncodedMHMessage(BaseAsyncMessage, EncodedMHMessage): - """Encoded MH Message.""" - - -class AsyncJSONMessage(BaseAsyncMessage, JSONMessage): - """Json serialized message.""" diff --git a/kombu/async/aws/sqs/queue.py b/kombu/async/aws/sqs/queue.py index 140ff31ac..c0f8c0f3f 100644 --- a/kombu/async/aws/sqs/queue.py +++ b/kombu/async/aws/sqs/queue.py @@ -4,7 +4,6 @@ from vine import transform -from .ext import Queue as _Queue from .message import AsyncMessage _all__ = ['AsyncQueue'] @@ -15,7 +14,7 @@ def list_first(rs): return rs[0] if len(rs) == 1 else None -class AsyncQueue(_Queue): +class AsyncQueue(): """Async SQS Queue.""" def __init__(self, connection=None, url=None, message_class=AsyncMessage): diff --git a/kombu/transport/SQS.py b/kombu/transport/SQS.py index 1ef4dcfcc..e41eab870 100644 --- a/kombu/transport/SQS.py +++ b/kombu/transport/SQS.py @@ -47,7 +47,7 @@ from kombu.async.aws import sqs as _asynsqs from kombu.async.aws.ext import boto3, exceptions from kombu.async.aws.sqs.connection import AsyncSQSConnection -from kombu.async.aws.sqs.message import Message +from kombu.async.aws.sqs.message import AsyncMessage from kombu.five import Empty, range, string_t, text_t from kombu.log import get_logger from kombu.utils import scheduling From aebabd8a178b8e41ecbec034f38a5f046477225a Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 09:50:39 -0800 Subject: [PATCH 20/44] removing tests that are no longer valid with boto3/SQS --- requirements/test-ci.txt | 1 + t/unit/async/aws/sqs/test_connection.py | 4 +- t/unit/async/aws/sqs/test_sqs.py | 34 ---------- t/unit/async/aws/test_connection.py | 85 +++++-------------------- 4 files changed, 20 insertions(+), 104 deletions(-) delete mode 100644 t/unit/async/aws/sqs/test_sqs.py diff --git a/requirements/test-ci.txt b/requirements/test-ci.txt index 0684adfcf..f2318d5a7 100644 --- a/requirements/test-ci.txt +++ b/requirements/test-ci.txt @@ -3,3 +3,4 @@ codecov redis PyYAML msgpack-python>0.2.0 +-r extras/sqs.txt diff --git a/t/unit/async/aws/sqs/test_connection.py b/t/unit/async/aws/sqs/test_connection.py index 9f1866cfc..cecd7f5ec 100644 --- a/t/unit/async/aws/sqs/test_connection.py +++ b/t/unit/async/aws/sqs/test_connection.py @@ -6,7 +6,7 @@ from case import Mock from kombu.async.aws.sqs.connection import ( - AsyncSQSConnection, Attributes, BatchResults, + AsyncSQSConnection, ) from kombu.async.aws.sqs.message import AsyncMessage from kombu.async.aws.sqs.queue import AsyncQueue @@ -72,7 +72,7 @@ def test_get_queue_attributes(self): ) self.x.get_object.assert_called_with( 'GetQueueAttributes', {'AttributeName': 'QueueSize'}, - Attributes, queue.id, callback=self.callback, + queue.id, callback=self.callback, ) def test_set_queue_attribute(self): diff --git a/t/unit/async/aws/sqs/test_sqs.py b/t/unit/async/aws/sqs/test_sqs.py deleted file mode 100644 index ea58596d0..000000000 --- a/t/unit/async/aws/sqs/test_sqs.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import, unicode_literals - -import pytest - -from case import Mock, patch - -from kombu.async.aws.sqs import regions, connect_to_region -from kombu.async.aws.sqs.connection import AsyncSQSConnection - -from ..case import AWSCase - - -class test_connect_to_region(AWSCase): - - def test_when_no_boto_installed(self, patching): - patching('kombu.async.aws.sqs.boto', None) - with pytest.raises(ImportError): - regions() - - def test_using_async_connection(self): - for region in regions(): - assert region.connection_cls is AsyncSQSConnection - - def test_connect_to_region(self): - with patch('kombu.async.aws.sqs.regions') as regions: - region = Mock(name='region') - region.name = 'us-west-1' - regions.return_value = [region] - conn = connect_to_region('us-west-1', kw=3.33) - assert conn is region.connect.return_value - region.connect.assert_called_with(kw=3.33) - - assert connect_to_region('foo') is None diff --git a/t/unit/async/aws/test_connection.py b/t/unit/async/aws/test_connection.py index 5de76aa5c..3a0e86316 100644 --- a/t/unit/async/aws/test_connection.py +++ b/t/unit/async/aws/test_connection.py @@ -13,11 +13,9 @@ from kombu.async import http from kombu.async.aws.connection import ( - AsyncHTTPConnection, AsyncHTTPSConnection, AsyncHTTPResponse, AsyncConnection, - AsyncAWSAuthConnection, AsyncAWSQueryConnection, ) @@ -38,21 +36,21 @@ def side_effect(ret): return m -class test_AsyncHTTPConnection(AWSCase): +class test_AsyncHTTPSConnection(AWSCase): def test_AsyncHTTPSConnection(self): x = AsyncHTTPSConnection('aws.vandelay.com') assert x.scheme == 'https' def test_http_client(self): - x = AsyncHTTPConnection('aws.vandelay.com') + x = AsyncHTTPSConnection('aws.vandelay.com') assert x.http_client is http.get_client() client = Mock(name='http_client') - y = AsyncHTTPConnection('aws.vandelay.com', http_client=client) + y = AsyncHTTPSConnection('aws.vandelay.com', http_client=client) assert y.http_client is client def test_args(self): - x = AsyncHTTPConnection( + x = AsyncHTTPSConnection( 'aws.vandelay.com', 8083, strict=True, timeout=33.3, ) assert x.host == 'aws.vandelay.com' @@ -62,13 +60,13 @@ def test_args(self): assert x.scheme == 'http' def test_request(self): - x = AsyncHTTPConnection('aws.vandelay.com') + x = AsyncHTTPSConnection('aws.vandelay.com') x.request('PUT', '/importer-exporter') assert x.path == '/importer-exporter' assert x.method == 'PUT' def test_request_with_body_buffer(self): - x = AsyncHTTPConnection('aws.vandelay.com') + x = AsyncHTTPSConnection('aws.vandelay.com') body = Mock(name='body') body.read.return_value = 'Vandelay Industries' x.request('PUT', '/importer-exporter', body) @@ -78,14 +76,14 @@ def test_request_with_body_buffer(self): body.read.assert_called_with() def test_request_with_body_text(self): - x = AsyncHTTPConnection('aws.vandelay.com') + x = AsyncHTTPSConnection('aws.vandelay.com') x.request('PUT', '/importer-exporter', 'Vandelay Industries') assert x.method == 'PUT' assert x.path == '/importer-exporter' assert x.body == 'Vandelay Industries' def test_request_with_headers(self): - x = AsyncHTTPConnection('aws.vandelay.com') + x = AsyncHTTPSConnection('aws.vandelay.com') headers = {'Proxy': 'proxy.vandelay.com'} x.request('PUT', '/importer-exporter', None, headers) assert 'Proxy' in dict(x.headers) @@ -106,7 +104,7 @@ def test_getrequest_AsyncHTTPSConnection(self): self.assert_request_created_with('https://aws.vandelay.com/', x) def test_getrequest_nondefault_port(self): - x = AsyncHTTPConnection('aws.vandelay.com', port=8080) + x = AsyncHTTPSConnection('aws.vandelay.com', port=8080) x.Request = Mock(name='Request') x.getrequest() self.assert_request_created_with('http://aws.vandelay.com:8080/', x) @@ -119,7 +117,7 @@ def test_getrequest_nondefault_port(self): def test_getresponse(self): client = Mock(name='client') client.add_request = passthrough(name='client.add_request') - x = AsyncHTTPConnection('aws.vandelay.com', http_client=client) + x = AsyncHTTPSConnection('aws.vandelay.com', http_client=client) x.Response = Mock(name='x.Response') request = x.getresponse() x.http_client.add_request.assert_called_with(request) @@ -134,7 +132,7 @@ def test_getresponse__real_response(self): client = Mock(name='client') client.add_request = passthrough(name='client.add_request') callback = PromiseMock(name='callback') - x = AsyncHTTPConnection('aws.vandelay.com', http_client=client) + x = AsyncHTTPSConnection('aws.vandelay.com', http_client=client) request = x.getresponse(callback) x.http_client.add_request.assert_called_with(request) @@ -157,16 +155,16 @@ def test_getresponse__real_response(self): assert repr(wresponse) def test_repr(self): - assert repr(AsyncHTTPConnection('aws.vandelay.com')) + assert repr(AsyncHTTPSConnection('aws.vandelay.com')) def test_putrequest(self): - x = AsyncHTTPConnection('aws.vandelay.com') + x = AsyncHTTPSConnection('aws.vandelay.com') x.putrequest('UPLOAD', '/new') assert x.method == 'UPLOAD' assert x.path == '/new' def test_putheader(self): - x = AsyncHTTPConnection('aws.vandelay.com') + x = AsyncHTTPSConnection('aws.vandelay.com') x.putheader('X-Foo', 'bar') assert x.headers == [('X-Foo', 'bar')] x.putheader('X-Bar', 'baz') @@ -176,14 +174,14 @@ def test_putheader(self): ] def test_send(self): - x = AsyncHTTPConnection('aws.vandelay.com') + x = AsyncHTTPSConnection('aws.vandelay.com') x.send('foo') assert x.body == 'foo' x.send('bar') assert x.body == 'foobar' def test_interface(self): - x = AsyncHTTPConnection('aws.vandelay.com') + x = AsyncHTTPSConnection('aws.vandelay.com') assert x.set_debuglevel(3) is None assert x.connect() is None assert x.close() is None @@ -220,7 +218,7 @@ def test_get_http_connection(self): x = AsyncConnection(client=Mock(name='client')) assert isinstance( x.get_http_connection('aws.vandelay.com', 80, False), - AsyncHTTPConnection, + AsyncHTTPSConnection, ) assert isinstance( x.get_http_connection('aws.vandelay.com', 443, True), @@ -233,55 +231,6 @@ def test_get_http_connection(self): assert conn.port == 80 -class test_AsyncAWSAuthConnection(AWSCase): - - @patch('boto.log', create=True) - def test_make_request(self, _): - x = AsyncAWSAuthConnection('aws.vandelay.com', - http_client=Mock(name='client')) - Conn = x.get_http_connection = Mock(name='get_http_connection') - callback = PromiseMock(name='callback') - ret = x.make_request('GET', '/foo', callback=callback) - assert ret is callback - Conn.return_value.request.assert_called() - Conn.return_value.getresponse.assert_called_with( - callback=callback, - ) - - @patch('boto.log', create=True) - def test_mexe(self, _): - x = AsyncAWSAuthConnection('aws.vandelay.com', - http_client=Mock(name='client')) - Conn = x.get_http_connection = Mock(name='get_http_connection') - request = x.build_base_http_request('GET', 'foo', '/auth') - callback = PromiseMock(name='callback') - x._mexe(request, callback=callback) - Conn.return_value.request.assert_called_with( - request.method, request.path, request.body, request.headers, - ) - Conn.return_value.getresponse.assert_called_with( - callback=callback, - ) - - no_callback_ret = x._mexe(request) - # _mexe always returns promise - assert isinstance(no_callback_ret, Thenable) - - @patch('boto.log', create=True) - def test_mexe__with_sender(self, _): - x = AsyncAWSAuthConnection('aws.vandelay.com', - http_client=Mock(name='client')) - Conn = x.get_http_connection = Mock(name='get_http_connection') - request = x.build_base_http_request('GET', 'foo', '/auth') - sender = Mock(name='sender') - callback = PromiseMock(name='callback') - x._mexe(request, sender=sender, callback=callback) - sender.assert_called_with( - Conn.return_value, request.method, request.path, - request.body, request.headers, callback, - ) - - class test_AsyncAWSQueryConnection(AWSCase): def setup(self): From 17f4ab00c3ad9575a8c97e183b841a44dd00c37a Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 12:26:28 -0800 Subject: [PATCH 21/44] fix boto3 dep, min version and no botocore --- requirements/extras/sqs.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements/extras/sqs.txt b/requirements/extras/sqs.txt index d278da8f6..3d6d07a93 100644 --- a/requirements/extras/sqs.txt +++ b/requirements/extras/sqs.txt @@ -1,3 +1,2 @@ -boto3 -botocore +boto3>=1.4.4 pycurl From a446e9d5f0610531b195cdb8f78affb42b99d716 Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 12:27:18 -0800 Subject: [PATCH 22/44] no boto2 for test --- t/unit/async/aws/case.py | 1 - 1 file changed, 1 deletion(-) diff --git a/t/unit/async/aws/case.py b/t/unit/async/aws/case.py index 9bfce209a..70d9565f8 100644 --- a/t/unit/async/aws/case.py +++ b/t/unit/async/aws/case.py @@ -7,7 +7,6 @@ @skip.if_pypy() -@skip.unless_module('boto') @skip.unless_module('boto3') @skip.unless_module('pycurl') @pytest.mark.usefixtures('hub') From b4911a497ea0cf4894e704b81b61a09b1cf35362 Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 13:30:40 -0800 Subject: [PATCH 23/44] cleaning up some SQS tests. fixing header parsing of response to msg --- kombu/async/aws/connection.py | 14 ++---- t/unit/async/aws/sqs/test_connection.py | 10 ++-- t/unit/async/aws/test_aws.py | 2 +- t/unit/async/aws/test_connection.py | 64 ++++++++++--------------- 4 files changed, 35 insertions(+), 55 deletions(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index 8e7339c4d..a102e7774 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -4,8 +4,6 @@ from vine import promise, transform -from io import BytesIO - from botocore.awsrequest import AWSRequest from botocore.response import get_response @@ -19,12 +17,12 @@ from xml.sax import parseString as sax_parse # noqa try: # pragma: no cover - from email import message_from_file + from email import message_from_bytes from email.mime.message import MIMEMessage except ImportError: # pragma: no cover from mimetools import Message as MIMEMessage # noqa - def message_from_file(m): # noqa + def message_from_bytes(m): # noqa return m __all__ = [ @@ -32,7 +30,6 @@ def message_from_file(m): # noqa ] -@python_2_unicode_compatible class AsyncHTTPResponse(object): """Async HTTP Response.""" @@ -53,11 +50,8 @@ def getheaders(self): @property def msg(self): if self._msg is None: - self._msg = MIMEMessage(message_from_file( - BytesIO(b'\r\n'.join( - b'{0}: {1}'.format(*h) for h in self.getheaders()) - ) - )) + bs = "\r\n".join("{}: {}".format(*h) for h in self.getheaders()) + self._msg = MIMEMessage(message_from_bytes(bs.encode())) return self._msg @property diff --git a/t/unit/async/aws/sqs/test_connection.py b/t/unit/async/aws/sqs/test_connection.py index cecd7f5ec..2959f17f5 100644 --- a/t/unit/async/aws/sqs/test_connection.py +++ b/t/unit/async/aws/sqs/test_connection.py @@ -180,7 +180,7 @@ def test_delete_message_batch(self): 'DeleteMessageBatchRequestEntry.2.Id': '2', 'DeleteMessageBatchRequestEntry.2.ReceiptHandle': 'r2', }, - BatchResults, queue.id, verb='POST', callback=self.callback, + queue.id, verb='POST', callback=self.callback, ) def test_send_message(self): @@ -188,7 +188,7 @@ def test_send_message(self): self.x.send_message(queue, 'hello', callback=self.callback) self.x.get_object.assert_called_with( 'SendMessage', {'MessageBody': 'hello'}, - AsyncMessage, queue.id, verb='POST', callback=self.callback, + queue.id, verb='POST', callback=self.callback, ) def test_send_message__with_delay_seconds(self): @@ -198,7 +198,7 @@ def test_send_message__with_delay_seconds(self): ) self.x.get_object.assert_called_with( 'SendMessage', {'MessageBody': 'hello', 'DelaySeconds': 303}, - AsyncMessage, queue.id, verb='POST', callback=self.callback, + queue.id, verb='POST', callback=self.callback, ) def test_send_message_batch(self): @@ -218,7 +218,7 @@ def test_send_message_batch(self): 'SendMessageBatchRequestEntry.2.MessageBody': 'B', 'SendMessageBatchRequestEntry.2.DelaySeconds': 303, }, - BatchResults, queue.id, verb='POST', callback=self.callback, + queue.id, verb='POST', callback=self.callback, ) def test_change_message_visibility(self): @@ -256,7 +256,7 @@ def preamble(n): preamble('2.ReceiptHandle'): 'r2', preamble('2.VisibilityTimeout'): 909, }, - BatchResults, queue.id, verb='POST', callback=self.callback, + queue.id, verb='POST', callback=self.callback, ) def test_get_all_queues(self): diff --git a/t/unit/async/aws/test_aws.py b/t/unit/async/aws/test_aws.py index f5ed9aeff..29c535abb 100644 --- a/t/unit/async/aws/test_aws.py +++ b/t/unit/async/aws/test_aws.py @@ -13,4 +13,4 @@ class test_connect_sqs(AWSCase): def test_connection(self): x = connect_sqs('AAKI', 'ASAK', http_client=Mock()) assert x - assert x.connection + assert x.sqs_connection diff --git a/t/unit/async/aws/test_connection.py b/t/unit/async/aws/test_connection.py index 3a0e86316..7c25a61c7 100644 --- a/t/unit/async/aws/test_connection.py +++ b/t/unit/async/aws/test_connection.py @@ -7,6 +7,7 @@ from case import Mock, patch from vine.abstract import Thenable +import boto3 from kombu.exceptions import HttpError from kombu.five import WhateverIO @@ -39,25 +40,23 @@ def side_effect(ret): class test_AsyncHTTPSConnection(AWSCase): def test_AsyncHTTPSConnection(self): - x = AsyncHTTPSConnection('aws.vandelay.com') + x = AsyncHTTPSConnection() assert x.scheme == 'https' def test_http_client(self): - x = AsyncHTTPSConnection('aws.vandelay.com') + x = AsyncHTTPSConnection() assert x.http_client is http.get_client() client = Mock(name='http_client') - y = AsyncHTTPSConnection('aws.vandelay.com', http_client=client) + y = AsyncHTTPSConnection(http_client=client) assert y.http_client is client def test_args(self): x = AsyncHTTPSConnection( - 'aws.vandelay.com', 8083, strict=True, timeout=33.3, + strict=True, timeout=33.3, ) - assert x.host == 'aws.vandelay.com' - assert x.port == 8083 assert x.strict assert x.timeout == 33.3 - assert x.scheme == 'http' + assert x.scheme == 'https' def test_request(self): x = AsyncHTTPSConnection('aws.vandelay.com') @@ -83,7 +82,7 @@ def test_request_with_body_text(self): assert x.body == 'Vandelay Industries' def test_request_with_headers(self): - x = AsyncHTTPSConnection('aws.vandelay.com') + x = AsyncHTTPSConnection() headers = {'Proxy': 'proxy.vandelay.com'} x.request('PUT', '/importer-exporter', None, headers) assert 'Proxy' in dict(x.headers) @@ -97,27 +96,10 @@ def assert_request_created_with(self, url, conn): validate_cert=VALIDATES_CERT, ) - def test_getrequest_AsyncHTTPSConnection(self): - x = AsyncHTTPSConnection('aws.vandelay.com') - x.Request = Mock(name='Request') - x.getrequest() - self.assert_request_created_with('https://aws.vandelay.com/', x) - - def test_getrequest_nondefault_port(self): - x = AsyncHTTPSConnection('aws.vandelay.com', port=8080) - x.Request = Mock(name='Request') - x.getrequest() - self.assert_request_created_with('http://aws.vandelay.com:8080/', x) - - y = AsyncHTTPSConnection('aws.vandelay.com', port=8443) - y.Request = Mock(name='Request') - y.getrequest() - self.assert_request_created_with('https://aws.vandelay.com:8443/', y) - def test_getresponse(self): client = Mock(name='client') client.add_request = passthrough(name='client.add_request') - x = AsyncHTTPSConnection('aws.vandelay.com', http_client=client) + x = AsyncHTTPSConnection(http_client=client) x.Response = Mock(name='x.Response') request = x.getresponse() x.http_client.add_request.assert_called_with(request) @@ -132,7 +114,7 @@ def test_getresponse__real_response(self): client = Mock(name='client') client.add_request = passthrough(name='client.add_request') callback = PromiseMock(name='callback') - x = AsyncHTTPSConnection('aws.vandelay.com', http_client=client) + x = AsyncHTTPSConnection(http_client=client) request = x.getresponse(callback) x.http_client.add_request.assert_called_with(request) @@ -149,22 +131,22 @@ def test_getresponse__real_response(self): assert wresponse.read() == 'The quick brown fox jumps' assert wresponse.status == 200 assert wresponse.getheader('X-Foo') == 'Hello' - assert dict(wresponse.getheaders()) == headers - assert wresponse.msg + headers_dict = wresponse.getheaders() + assert dict(headers_dict) == headers assert wresponse.msg assert repr(wresponse) def test_repr(self): - assert repr(AsyncHTTPSConnection('aws.vandelay.com')) + assert repr(AsyncHTTPSConnection()) def test_putrequest(self): - x = AsyncHTTPSConnection('aws.vandelay.com') + x = AsyncHTTPSConnection() x.putrequest('UPLOAD', '/new') assert x.method == 'UPLOAD' assert x.path == '/new' def test_putheader(self): - x = AsyncHTTPSConnection('aws.vandelay.com') + x = AsyncHTTPSConnection() x.putheader('X-Foo', 'bar') assert x.headers == [('X-Foo', 'bar')] x.putheader('X-Bar', 'baz') @@ -174,14 +156,14 @@ def test_putheader(self): ] def test_send(self): - x = AsyncHTTPSConnection('aws.vandelay.com') + x = AsyncHTTPSConnection() x.send('foo') assert x.body == 'foo' x.send('bar') assert x.body == 'foobar' def test_interface(self): - x = AsyncHTTPSConnection('aws.vandelay.com') + x = AsyncHTTPSConnection() assert x.set_debuglevel(3) is None assert x.connect() is None assert x.close() is None @@ -217,15 +199,15 @@ def test_client(self): def test_get_http_connection(self): x = AsyncConnection(client=Mock(name='client')) assert isinstance( - x.get_http_connection('aws.vandelay.com', 80, False), + x.get_http_connection(False), AsyncHTTPSConnection, ) assert isinstance( - x.get_http_connection('aws.vandelay.com', 443, True), + x.get_http_connection(True), AsyncHTTPSConnection, ) - conn = x.get_http_connection('aws.vandelay.com', 80, False) + conn = x.get_http_connection(False) assert conn.http_client is x._httpclient assert conn.host == 'aws.vandelay.com' assert conn.port == 80 @@ -234,8 +216,12 @@ def test_get_http_connection(self): class test_AsyncAWSQueryConnection(AWSCase): def setup(self): - self.x = AsyncAWSQueryConnection('aws.vandelay.com', - http_client=Mock(name='client')) + session = boto3.session.Session( + aws_access_key_id='AAA', + aws_secret_access_key='AAAA', + ) + sqs_client = session.client('sqs') + self.x = AsyncAWSQueryConnection(sqs_client, http_client=Mock(name='client')) @patch('boto.log', create=True) def test_make_request(self, _): From 5729ff8f0e3bfb892dcf64bd38b2e4559f0917ce Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 16:48:09 -0800 Subject: [PATCH 24/44] fixing some sqs tests --- kombu/async/aws/connection.py | 16 +++++++--- kombu/async/aws/ext.py | 6 +++- t/unit/async/aws/sqs/test_connection.py | 8 ++--- t/unit/async/aws/test_connection.py | 42 +++++++++++-------------- 4 files changed, 38 insertions(+), 34 deletions(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index a102e7774..558bd2d32 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -4,8 +4,7 @@ from vine import promise, transform -from botocore.awsrequest import AWSRequest -from botocore.response import get_response +from kombu.async.aws.ext import AWSRequest, get_response from kombu.async.http import Headers, Request, get_client from kombu.five import items, python_2_unicode_compatible @@ -175,10 +174,19 @@ def make_request(self, operation, params_, path, verb, callback=None): if operation: params['Action'] = operation signer = self.sqs_connection._request_signer - request = AWSRequest(method=verb, url=path, data=params) - signing_type = 'presignurl' if verb == 'GET' else 'standard' + + # defaults for non-get + signing_type = 'standard' + param_payload = {'data': params} + if verb.lower() == 'get': + # query-based opts + signing_type = 'presignurl' + param_payload = {'params': params} + + request = AWSRequest(method=verb, url=path, **param_payload) signer.sign(operation, request, signing_type=signing_type) prepared_request = request.prepare() + # print(prepared_request.url) # print(prepared_request.headers) # print(prepared_request.body) diff --git a/kombu/async/aws/ext.py b/kombu/async/aws/ext.py index 2c7e7ed3b..4268073d3 100644 --- a/kombu/async/aws/ext.py +++ b/kombu/async/aws/ext.py @@ -6,6 +6,8 @@ import boto3 from botocore import exceptions from boto3 import session + from botocore.awsrequest import AWSRequest + from botocore.response import get_response except ImportError: boto3 = session = None @@ -16,8 +18,10 @@ class BotoCoreError(Exception): pass exceptions = _void() exceptions.BotoCoreError = BotoCoreError + AWSRequest = _void() + get_response = _void() __all__ = [ - 'exceptions' + 'exceptions', 'AWSRequest', 'get_response' ] diff --git a/t/unit/async/aws/sqs/test_connection.py b/t/unit/async/aws/sqs/test_connection.py index 2959f17f5..bc1fb90cf 100644 --- a/t/unit/async/aws/sqs/test_connection.py +++ b/t/unit/async/aws/sqs/test_connection.py @@ -23,17 +23,15 @@ def setup(self): self.x = AsyncSQSConnection('ak', 'sk', http_client=Mock()) self.x.get_object = Mock(name='X.get_object') self.x.get_status = Mock(name='X.get_status') - self.x.get_list = Mock(nanme='X.get_list') + self.x.get_list = Mock(name='X.get_list') self.callback = PromiseMock(name='callback') def test_without_boto(self): from kombu.async.aws.sqs import connection - prev, connection.boto = connection.boto, None + sqs = Mock(name='sqs') try: with pytest.raises(ImportError): - AsyncSQSConnection('ak', 'sk', http_client=Mock()) - finally: - connection.boto = prev + AsyncSQSConnection(sqs, 'ak', 'sk', http_client=Mock()) def test_default_region(self): assert self.x.region diff --git a/t/unit/async/aws/test_connection.py b/t/unit/async/aws/test_connection.py index 7c25a61c7..bf0da7f51 100644 --- a/t/unit/async/aws/test_connection.py +++ b/t/unit/async/aws/test_connection.py @@ -24,6 +24,11 @@ from t.mocks import PromiseMock +try: + from urllib.parse import urlparse, parse_qs +except ImportError: + from urlparse import urlparse, parse_qs # noqa + # Not currently working VALIDATES_CERT = False @@ -184,33 +189,23 @@ def test_with_error(self): class test_AsyncConnection(AWSCase): - def test_when_boto_missing(self, patching): - patching('kombu.async.aws.connection.boto', None) - with pytest.raises(ImportError): - AsyncConnection(Mock(name='client')) - def test_client(self): - x = AsyncConnection() + sqs = Mock(name='sqs') + x = AsyncConnection(sqs) assert x._httpclient is http.get_client() client = Mock(name='client') - y = AsyncConnection(http_client=client) + y = AsyncConnection(sqs, http_client=client) assert y._httpclient is client def test_get_http_connection(self): - x = AsyncConnection(client=Mock(name='client')) + sqs = Mock(name='sqs') + x = AsyncConnection(sqs) assert isinstance( - x.get_http_connection(False), + x.get_http_connection(), AsyncHTTPSConnection, ) - assert isinstance( - x.get_http_connection(True), - AsyncHTTPSConnection, - ) - - conn = x.get_http_connection(False) + conn = x.get_http_connection() assert conn.http_client is x._httpclient - assert conn.host == 'aws.vandelay.com' - assert conn.port == 80 class test_AsyncAWSQueryConnection(AWSCase): @@ -223,18 +218,18 @@ def setup(self): sqs_client = session.client('sqs') self.x = AsyncAWSQueryConnection(sqs_client, http_client=Mock(name='client')) - @patch('boto.log', create=True) - def test_make_request(self, _): + def test_make_request(self): _mexe, self.x._mexe = self.x._mexe, Mock(name='_mexe') Conn = self.x.get_http_connection = Mock(name='get_http_connection') callback = PromiseMock(name='callback') self.x.make_request( - 'action', {'foo': 1}, '/', 'GET', callback=callback, + 'action', {'foo': 1}, 'https://foo.com/', 'GET', callback=callback, ) self.x._mexe.assert_called() request = self.x._mexe.call_args[0][0] - assert request.params['Action'] == 'action' - assert request.params['Version'] == self.x.APIVersion + parsed = urlparse(request.url) + params = parse_qs(parsed.query) + assert params['Action'][0] == 'action' ret = _mexe(request, callback=callback) assert ret is callback @@ -243,8 +238,7 @@ def test_make_request(self, _): callback=callback, ) - @patch('boto.log', create=True) - def test_make_request__no_action(self, _): + def test_make_request__no_action(self): self.x._mexe = Mock(name='_mexe') self.x.get_http_connection = Mock(name='get_http_connection') callback = PromiseMock(name='callback') From 07bf7098b096333cc02efd2a116db563ed03b49f Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 16:52:21 -0800 Subject: [PATCH 25/44] removing response-parsing tests that are no longer necessary as we're using the botocore response parsing machinery instead of implementing SAX parsing in kombu. --- t/unit/async/aws/test_connection.py | 105 ++-------------------------- 1 file changed, 4 insertions(+), 101 deletions(-) diff --git a/t/unit/async/aws/test_connection.py b/t/unit/async/aws/test_connection.py index bf0da7f51..8ea8ecbfb 100644 --- a/t/unit/async/aws/test_connection.py +++ b/t/unit/async/aws/test_connection.py @@ -243,23 +243,13 @@ def test_make_request__no_action(self): self.x.get_http_connection = Mock(name='get_http_connection') callback = PromiseMock(name='callback') self.x.make_request( - None, {'foo': 1}, '/', 'GET', callback=callback, + None, {'foo': 1}, 'http://foo.com/', 'GET', callback=callback, ) self.x._mexe.assert_called() request = self.x._mexe.call_args[0][0] - assert 'Action' not in request.params - assert request.params['Version'] == self.x.APIVersion - - @contextmanager - def mock_sax_parse(self, parser): - with patch('kombu.async.aws.connection.sax_parse') as sax_parse: - with patch('kombu.async.aws.connection.XmlHandler') as xh: - - def effect(body, h): - return parser(xh.call_args[0][0], body, h) - sax_parse.side_effect = effect - yield (sax_parse, xh) - sax_parse.assert_called() + parsed = urlparse(request.url) + params = parse_qs(parsed.query) + assert 'Action' not in params def Response(self, status, body): r = Mock(name='response') @@ -276,90 +266,3 @@ def mock_make_request(self): def assert_make_request_called(self): self.x.make_request.assert_called() return self.x.make_request.call_args[1]['callback'] - - def test_get_list(self): - with self.mock_make_request() as callback: - self.x.get_list('action', {'p': 3.3}, ['m'], callback=callback) - on_ready = self.assert_make_request_called() - - def parser(dest, body, h): - dest.append('hi') - dest.append('there') - - with self.mock_sax_parse(parser): - on_ready(self.Response(200, 'hello')) - callback.assert_called_with(['hi', 'there']) - - def test_get_list_error(self): - with self.mock_make_request() as callback: - self.x.get_list('action', {'p': 3.3}, ['m'], callback=callback) - on_ready = self.assert_make_request_called() - - with pytest.raises(self.x.ResponseError): - on_ready(self.Response(404, 'Not found')) - - def test_get_object(self): - with self.mock_make_request() as callback: - - class Result(object): - parent = None - value = None - - def __init__(self, parent): - self.parent = parent - - self.x.get_object('action', {'p': 3.3}, Result, callback=callback) - on_ready = self.assert_make_request_called() - - def parser(dest, body, h): - dest.value = 42 - - with self.mock_sax_parse(parser): - on_ready(self.Response(200, 'hello')) - - callback.assert_called() - result = callback.call_args[0][0] - assert result.value == 42 - assert result.parent - - def test_get_object_error(self): - with self.mock_make_request() as callback: - self.x.get_object('action', {'p': 3.3}, object, callback=callback) - on_ready = self.assert_make_request_called() - - with pytest.raises(self.x.ResponseError): - on_ready(self.Response(404, 'Not found')) - - def test_get_status(self): - with self.mock_make_request() as callback: - self.x.get_status('action', {'p': 3.3}, callback=callback) - on_ready = self.assert_make_request_called() - set_status_to = [True] - - def parser(dest, body, b): - dest.status = set_status_to[0] - - with self.mock_sax_parse(parser): - on_ready(self.Response(200, 'hello')) - callback.assert_called_with(True) - - set_status_to[0] = False - with self.mock_sax_parse(parser): - on_ready(self.Response(200, 'hello')) - callback.assert_called_with(False) - - def test_get_status_error(self): - with self.mock_make_request() as callback: - self.x.get_status('action', {'p': 3.3}, callback=callback) - on_ready = self.assert_make_request_called() - - with pytest.raises(self.x.ResponseError): - on_ready(self.Response(404, 'Not found')) - - def test_get_status_error_empty_body(self): - with self.mock_make_request() as callback: - self.x.get_status('action', {'p': 3.3}, callback=callback) - on_ready = self.assert_make_request_called() - - with pytest.raises(self.x.ResponseError): - on_ready(self.Response(200, '')) From b28bc7c643a8e461fbce121469dce998d2038868 Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 17:25:56 -0800 Subject: [PATCH 26/44] fixing more SQS tests --- kombu/async/aws/sqs/connection.py | 5 +++- t/unit/async/aws/sqs/test_connection.py | 34 ++++++++++++------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/kombu/async/aws/sqs/connection.py b/kombu/async/aws/sqs/connection.py index 32a94af86..cbd07d38a 100644 --- a/kombu/async/aws/sqs/connection.py +++ b/kombu/async/aws/sqs/connection.py @@ -73,7 +73,10 @@ def receive_message(self, queue, if visibility_timeout: params['VisibilityTimeout'] = visibility_timeout if attributes: - self.build_list_params(params, attributes, 'AttributeName') + attrs = {} + for idx, attr in enumerate(attributes): + attrs['AttributeName.' + str(idx+1)] = attr + params.update(attrs) if wait_time_seconds is not None: params['WaitTimeSeconds'] = wait_time_seconds queue_url = self.get_queue_url(queue) diff --git a/t/unit/async/aws/sqs/test_connection.py b/t/unit/async/aws/sqs/test_connection.py index bc1fb90cf..99e18b8e2 100644 --- a/t/unit/async/aws/sqs/test_connection.py +++ b/t/unit/async/aws/sqs/test_connection.py @@ -3,11 +3,12 @@ import pytest -from case import Mock +from case import Mock, MagicMock from kombu.async.aws.sqs.connection import ( - AsyncSQSConnection, + AsyncSQSConnection ) +from kombu.async.aws.ext import boto3 from kombu.async.aws.sqs.message import AsyncMessage from kombu.async.aws.sqs.queue import AsyncQueue from kombu.utils.uuid import uuid @@ -20,27 +21,24 @@ class test_AsyncSQSConnection(AWSCase): def setup(self): - self.x = AsyncSQSConnection('ak', 'sk', http_client=Mock()) + session = boto3.session.Session( + aws_access_key_id='AAA', + aws_secret_access_key='AAAA', + ) + sqs_client = session.client('sqs') + self.x = AsyncSQSConnection(sqs_client, 'ak', 'sk', http_client=Mock()) self.x.get_object = Mock(name='X.get_object') self.x.get_status = Mock(name='X.get_status') self.x.get_list = Mock(name='X.get_list') self.callback = PromiseMock(name='callback') - def test_without_boto(self): - from kombu.async.aws.sqs import connection - sqs = Mock(name='sqs') - try: - with pytest.raises(ImportError): - AsyncSQSConnection(sqs, 'ak', 'sk', http_client=Mock()) + sqs_client.get_queue_url = MagicMock(return_value={'QueueUrl': 'http://aws.com'}) - def test_default_region(self): - assert self.x.region - assert issubclass(self.x.region.connection_cls, AsyncSQSConnection) def test_create_queue(self): self.x.create_queue('foo', callback=self.callback) self.x.get_object.assert_called_with( - 'CreateQueue', {'QueueName': 'foo'}, AsyncQueue, + 'CreateQueue', {'QueueName': 'foo'}, callback=self.callback, ) @@ -53,7 +51,7 @@ def test_create_queue__with_visibility_timeout(self): 'QueueName': 'foo', 'DefaultVisibilityTimeout': '33' }, - AsyncQueue, callback=self.callback + callback=self.callback ) def test_delete_queue(self): @@ -92,7 +90,7 @@ def test_receive_message(self): self.x.get_list.assert_called_with( 'ReceiveMessage', {'MaxNumberOfMessages': 4}, [('Message', AsyncMessage)], - queue, callback=self.callback, + 'http://aws.com', callback=self.callback, parent=queue, ) @@ -105,7 +103,7 @@ def test_receive_message__with_visibility_timeout(self): 'VisibilityTimeout': 3666, }, [('Message', AsyncMessage)], - queue, callback=self.callback, + 'http://aws.com', callback=self.callback, parent=queue, ) @@ -120,7 +118,7 @@ def test_receive_message__with_wait_time_seconds(self): 'WaitTimeSeconds': 303, }, [('Message', AsyncMessage)], - queue, callback=self.callback, + 'http://aws.com', callback=self.callback, parent=queue, ) @@ -136,7 +134,7 @@ def test_receive_message__with_attributes(self): 'MaxNumberOfMessages': 4, }, [('Message', AsyncMessage)], - queue, callback=self.callback, + 'http://aws.com', callback=self.callback, parent=queue, ) From e67ec195ef985d3af4b855127ce79cf5e7066f5b Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 17:31:31 -0800 Subject: [PATCH 27/44] wants a region --- t/unit/async/aws/sqs/test_connection.py | 1 + t/unit/async/aws/test_connection.py | 1 + 2 files changed, 2 insertions(+) diff --git a/t/unit/async/aws/sqs/test_connection.py b/t/unit/async/aws/sqs/test_connection.py index 99e18b8e2..5aafa8f66 100644 --- a/t/unit/async/aws/sqs/test_connection.py +++ b/t/unit/async/aws/sqs/test_connection.py @@ -24,6 +24,7 @@ def setup(self): session = boto3.session.Session( aws_access_key_id='AAA', aws_secret_access_key='AAAA', + region_name='us-west-2', ) sqs_client = session.client('sqs') self.x = AsyncSQSConnection(sqs_client, 'ak', 'sk', http_client=Mock()) diff --git a/t/unit/async/aws/test_connection.py b/t/unit/async/aws/test_connection.py index 8ea8ecbfb..f23e3104c 100644 --- a/t/unit/async/aws/test_connection.py +++ b/t/unit/async/aws/test_connection.py @@ -214,6 +214,7 @@ def setup(self): session = boto3.session.Session( aws_access_key_id='AAA', aws_secret_access_key='AAAA', + region_name='us-west-2', ) sqs_client = session.client('sqs') self.x = AsyncAWSQueryConnection(sqs_client, http_client=Mock(name='client')) From 78158a909b0378cde8805e3994b613f131fa2f1e Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 17:44:09 -0800 Subject: [PATCH 28/44] trying to fix py2 parsing of sqs message --- kombu/async/aws/connection.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index 558bd2d32..6b6b82bbd 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -9,26 +9,29 @@ from kombu.async.http import Headers, Request, get_client from kombu.five import items, python_2_unicode_compatible +import io + try: from urllib.parse import urlunsplit except ImportError: from urlparse import urlunsplit # noqa -from xml.sax import parseString as sax_parse # noqa try: # pragma: no cover from email import message_from_bytes from email.mime.message import MIMEMessage except ImportError: # pragma: no cover from mimetools import Message as MIMEMessage # noqa + from email import message_from_file def message_from_bytes(m): # noqa - return m + return message_from_file(io.BytesIO(m)) __all__ = [ 'AsyncHTTPSConnection', 'AsyncConnection', ] +@python_2_unicode_compatible class AsyncHTTPResponse(object): """Async HTTP Response.""" From 170898627d0103e3eff04b8e8d8df75f30a7465e Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 17:52:24 -0800 Subject: [PATCH 29/44] lint --- kombu/async/aws/connection.py | 27 +++++++++++++++++-------- kombu/async/aws/sqs/connection.py | 6 ++++-- kombu/async/aws/sqs/message.py | 4 +++- kombu/transport/SQS.py | 7 +++---- t/unit/async/aws/sqs/test_connection.py | 7 +++---- t/unit/async/aws/test_connection.py | 7 +++---- 6 files changed, 35 insertions(+), 23 deletions(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index 6b6b82bbd..734ff340e 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -169,8 +169,10 @@ def _mexe(self, request, sender=None, callback=None): class AsyncAWSQueryConnection(AsyncConnection): """Async AWS Query Connection.""" - def __init__(self, sqs_connection, http_client=None, http_client_params={}, **kwargs): - AsyncConnection.__init__(self, sqs_connection, http_client, **http_client_params) + def __init__(self, sqs_connection, http_client=None, http_client_params={}, + **kwargs): + AsyncConnection.__init__(self, sqs_connection, http_client, + **http_client_params) def make_request(self, operation, params_, path, verb, callback=None): params = params_.copy() @@ -199,8 +201,9 @@ def get_list(self, operation, params, markers, path='/', parent=None, verb='POST', callback=None): return self.make_request( operation, params, path, verb, - callback=transform( - self._on_list_ready, callback, parent or self, markers, operation + callback=transform( + self._on_list_ready, callback, parent or self, markers, + operation ), ) @@ -225,7 +228,9 @@ def get_status(self, operation, params, def _on_list_ready(self, parent, markers, operation, response): service_model = self.sqs_connection.meta.service_model if response.status == 200: - httpres, parsed = get_response(service_model.operation_model(operation), response.response) + httpres, parsed = get_response( + service_model.operation_model(operation), response.response + ) return parsed else: raise self._for_status(response, response.read()) @@ -233,7 +238,9 @@ def _on_list_ready(self, parent, markers, operation, response): def _on_obj_ready(self, parent, operation, response): service_model = self.sqs_connection.meta.service_model if response.status == 200: - httpres, parsed = get_response(service_model.operation_model(operation), response.response) + httpres, parsed = get_response( + service_model.operation_model(operation), response.response + ) return parsed else: raise self._for_status(response, response.read()) @@ -241,11 +248,15 @@ def _on_obj_ready(self, parent, operation, response): def _on_status_ready(self, parent, operation, response): service_model = self.sqs_connection.meta.service_model if response.status == 200: - httpres, parsed = get_response(service_model.operation_model(operation), response.response) + httpres, parsed = get_response( + service_model.operation_model(operation), response.response + ) return httpres['HTTPStatusCode'] else: raise self._for_status(response, response.read()) def _for_status(self, response, body): context = 'Empty body' if not body else 'HTTP Error' - return Exception("Request {} HTTP {} {} ({})".format(context, response.status, response.reason, body)) + return Exception("Request {} HTTP {} {} ({})".format( + context, response.status, response.reason, body + )) diff --git a/kombu/async/aws/sqs/connection.py b/kombu/async/aws/sqs/connection.py index cbd07d38a..02904afd4 100644 --- a/kombu/async/aws/sqs/connection.py +++ b/kombu/async/aws/sqs/connection.py @@ -17,7 +17,9 @@ class AsyncSQSConnection(AsyncAWSQueryConnection): """Async SQS Connection.""" - def __init__(self, sqs_connection, aws_access_key_id=None, aws_secret_access_key=None, + def __init__(self, sqs_connection, + aws_access_key_id=None, + aws_secret_access_key=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, region=None, *args, **kwargs): @@ -75,7 +77,7 @@ def receive_message(self, queue, if attributes: attrs = {} for idx, attr in enumerate(attributes): - attrs['AttributeName.' + str(idx+1)] = attr + attrs['AttributeName.' + str(idx + 1)] = attr params.update(attrs) if wait_time_seconds is not None: params['WaitTimeSeconds'] = wait_time_seconds diff --git a/kombu/async/aws/sqs/message.py b/kombu/async/aws/sqs/message.py index bd40667ff..fd25caeb6 100644 --- a/kombu/async/aws/sqs/message.py +++ b/kombu/async/aws/sqs/message.py @@ -2,8 +2,10 @@ """Amazon SQS message implementation.""" from __future__ import absolute_import, unicode_literals +from kombu.message import Message -class BaseAsyncMessage(object): + +class BaseAsyncMessage(Message): """Base class for messages received on async client.""" diff --git a/kombu/transport/SQS.py b/kombu/transport/SQS.py index e41eab870..b36abb643 100644 --- a/kombu/transport/SQS.py +++ b/kombu/transport/SQS.py @@ -44,7 +44,6 @@ from vine import transform, ensure_promise, promise from kombu.async import get_event_loop -from kombu.async.aws import sqs as _asynsqs from kombu.async.aws.ext import boto3, exceptions from kombu.async.aws.sqs.connection import AsyncSQSConnection from kombu.async.aws.sqs.message import AsyncMessage @@ -197,7 +196,7 @@ def _put(self, queue, message, **kwargs): """Put message onto queue.""" q_url = self._new_queue(queue) kwargs = {'QueueUrl': q_url, - 'MessageBody': Message().encode(dumps(message))} + 'MessageBody': AsyncMessage().encode(dumps(message))} if queue.endswith('.fifo'): if 'MessageGroupId' in message['properties']: kwargs['MessageGroupId'] = \ @@ -287,7 +286,7 @@ def _get_bulk(self, queue, if resp['Messages']: for m in resp['Messages']: - m['Body'] = Message().decode(m['Body']) + m['Body'] = AsyncMessage().decode(m['Body']) for msg in self._messages_to_python(resp['Messages'], queue): self.connection._deliver(msg, queue) return @@ -299,7 +298,7 @@ def _get(self, queue): resp = self.sqs.receive_message(q_url) if resp['Messages']: - body = Message().decode(resp['Messages'][0]['Body']) + body = AsyncMessage().decode(resp['Messages'][0]['Body']) resp['Messages'][0]['Body'] = body return self._messages_to_python(resp['Messages'], queue)[0] raise Empty() diff --git a/t/unit/async/aws/sqs/test_connection.py b/t/unit/async/aws/sqs/test_connection.py index 5aafa8f66..ade01868f 100644 --- a/t/unit/async/aws/sqs/test_connection.py +++ b/t/unit/async/aws/sqs/test_connection.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, unicode_literals -import pytest - from case import Mock, MagicMock from kombu.async.aws.sqs.connection import ( @@ -33,8 +31,9 @@ def setup(self): self.x.get_list = Mock(name='X.get_list') self.callback = PromiseMock(name='callback') - sqs_client.get_queue_url = MagicMock(return_value={'QueueUrl': 'http://aws.com'}) - + sqs_client.get_queue_url = MagicMock(return_value={ + 'QueueUrl': 'http://aws.com' + }) def test_create_queue(self): self.x.create_queue('foo', callback=self.callback) diff --git a/t/unit/async/aws/test_connection.py b/t/unit/async/aws/test_connection.py index f23e3104c..0c5fe2574 100644 --- a/t/unit/async/aws/test_connection.py +++ b/t/unit/async/aws/test_connection.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, unicode_literals -import pytest - from contextlib import contextmanager -from case import Mock, patch +from case import Mock from vine.abstract import Thenable import boto3 @@ -217,7 +215,8 @@ def setup(self): region_name='us-west-2', ) sqs_client = session.client('sqs') - self.x = AsyncAWSQueryConnection(sqs_client, http_client=Mock(name='client')) + self.x = AsyncAWSQueryConnection(sqs_client, + http_client=Mock(name='client')) def test_make_request(self): _mexe, self.x._mexe = self.x._mexe, Mock(name='_mexe') From fefd287e5c08a2425a14bde7a3d6e1949afe60d9 Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 17:59:58 -0800 Subject: [PATCH 30/44] py2/py2 message header parsing stupidness --- kombu/async/aws/connection.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index 734ff340e..9ec77bd25 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -19,12 +19,21 @@ try: # pragma: no cover from email import message_from_bytes from email.mime.message import MIMEMessage + + # py3 + def message_from_headers(hdr): # noqa + bs = "\r\n".join("{}: {}".format(*h) for h in hdr) + return message_from_bytes(bs.encode()) + except ImportError: # pragma: no cover - from mimetools import Message as MIMEMessage # noqa from email import message_from_file + from mimetools import Message as MIMEMessage # noqa - def message_from_bytes(m): # noqa - return message_from_file(io.BytesIO(m)) + # py2 + def message_from_headers(hdr): # noqa + return message_from_file(io.BytesIO(b'\r\n'.join( + b'{0}: {1}'.format(*h) for h in hdr + ))) __all__ = [ 'AsyncHTTPSConnection', 'AsyncConnection', @@ -52,8 +61,7 @@ def getheaders(self): @property def msg(self): if self._msg is None: - bs = "\r\n".join("{}: {}".format(*h) for h in self.getheaders()) - self._msg = MIMEMessage(message_from_bytes(bs.encode())) + self._msg = MIMEMessage(message_from_headers()) return self._msg @property From 56a38f0090f53a108c45c00ed00493a8e91ee6cd Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 18:02:10 -0800 Subject: [PATCH 31/44] forgot --- kombu/async/aws/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index 9ec77bd25..75e7051ad 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -61,7 +61,7 @@ def getheaders(self): @property def msg(self): if self._msg is None: - self._msg = MIMEMessage(message_from_headers()) + self._msg = MIMEMessage(message_from_headers(self.getheaders())) return self._msg @property From a8c551f7ca749328a4789ebc6855c5aed16b9c9c Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 18:05:44 -0800 Subject: [PATCH 32/44] python 2 sux --- kombu/async/aws/connection.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index 75e7051ad..da3734a1d 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -26,14 +26,13 @@ def message_from_headers(hdr): # noqa return message_from_bytes(bs.encode()) except ImportError: # pragma: no cover - from email import message_from_file from mimetools import Message as MIMEMessage # noqa # py2 def message_from_headers(hdr): # noqa - return message_from_file(io.BytesIO(b'\r\n'.join( + return io.BytesIO(b'\r\n'.join( b'{0}: {1}'.format(*h) for h in hdr - ))) + )) __all__ = [ 'AsyncHTTPSConnection', 'AsyncConnection', From 7c0409377079d0fe71c38a086696255031c80a6f Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 18:08:29 -0800 Subject: [PATCH 33/44] flake8 --- kombu/async/http/base.py | 2 +- kombu/async/http/curl.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/kombu/async/http/base.py b/kombu/async/http/base.py index 42c117dab..a8eb6edf0 100644 --- a/kombu/async/http/base.py +++ b/kombu/async/http/base.py @@ -200,7 +200,7 @@ def body(self): self._body = self.buffer.getvalue() return self._body - # these are for compatibility with Requests + # these are for compatibility with Requests @property def status_code(self): return self.code diff --git a/kombu/async/http/curl.py b/kombu/async/http/curl.py index e54cd0a1a..d8520ded3 100644 --- a/kombu/async/http/curl.py +++ b/kombu/async/http/curl.py @@ -177,9 +177,6 @@ def _process(self, curl, errno=None, reason=None, _pycurl=pycurl): request=request, code=code, headers=info['headers'], buffer=buffer, effective_url=effective_url, error=error, )) - # except Exception as exc: - # self.hub.on_callback_error(request.on_ready, exc) - # raise def _setup_request(self, curl, request, buffer, headers, _pycurl=pycurl): setopt = curl.setopt From b137cd31ee4a612357b0bbd7674491efcf443d55 Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Wed, 15 Feb 2017 18:15:10 -0800 Subject: [PATCH 34/44] Import boto3 from the right place --- t/unit/async/aws/test_connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/unit/async/aws/test_connection.py b/t/unit/async/aws/test_connection.py index 0c5fe2574..53b5e6d81 100644 --- a/t/unit/async/aws/test_connection.py +++ b/t/unit/async/aws/test_connection.py @@ -5,7 +5,6 @@ from case import Mock from vine.abstract import Thenable -import boto3 from kombu.exceptions import HttpError from kombu.five import WhateverIO @@ -17,6 +16,7 @@ AsyncConnection, AsyncAWSQueryConnection, ) +from kombu.async.aws.ext import boto3 from .case import AWSCase From 7377d71e937fcfee144400a7a43d85c12e32f399 Mon Sep 17 00:00:00 2001 From: Adam Szeptycki Date: Thu, 16 Feb 2017 17:55:08 +0100 Subject: [PATCH 35/44] Changes --- kombu/async/aws/connection.py | 2 +- kombu/async/aws/sqs/message.py | 8 +++++++- kombu/transport/SQS.py | 7 ++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index da3734a1d..8d7b16d1d 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -258,7 +258,7 @@ def _on_status_ready(self, parent, operation, response): httpres, parsed = get_response( service_model.operation_model(operation), response.response ) - return httpres['HTTPStatusCode'] + return httpres.code else: raise self._for_status(response, response.read()) diff --git a/kombu/async/aws/sqs/message.py b/kombu/async/aws/sqs/message.py index fd25caeb6..aedfa0afe 100644 --- a/kombu/async/aws/sqs/message.py +++ b/kombu/async/aws/sqs/message.py @@ -15,7 +15,13 @@ class AsyncRawMessage(BaseAsyncMessage): class AsyncMessage(BaseAsyncMessage): """Serialized message.""" - + + def encode(self, value): + return value + + def decode(self, value): + return value + def __getitem__(self, item): """Support Boto3-style access on a message.""" if item == 'ReceiptHandle': diff --git a/kombu/transport/SQS.py b/kombu/transport/SQS.py index b36abb643..411d5f6a7 100644 --- a/kombu/transport/SQS.py +++ b/kombu/transport/SQS.py @@ -370,12 +370,13 @@ def _restore(self, message, def basic_ack(self, delivery_tag, multiple=False): try: - message = self.qos.get(delivery_tag).delivery_info['sqs_message'] + message = self.qos.get(delivery_tag).delivery_info + sqs_message = message['sqs_message'] except KeyError: pass else: - self.asynsqs.delete_message(message['queue'], - message['ReceiptHandle']) + self.asynsqs.delete_message(message['sqs_queue'], + sqs_message['ReceiptHandle']) super(Channel, self).basic_ack(delivery_tag) def _size(self, queue): From 87d5f88b7e3f418de378a3d0f61eb6374de2d027 Mon Sep 17 00:00:00 2001 From: Adam Szeptycki Date: Thu, 16 Feb 2017 18:49:49 +0100 Subject: [PATCH 36/44] Update encode fuction --- kombu/async/aws/sqs/message.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/kombu/async/aws/sqs/message.py b/kombu/async/aws/sqs/message.py index aedfa0afe..893e4f369 100644 --- a/kombu/async/aws/sqs/message.py +++ b/kombu/async/aws/sqs/message.py @@ -3,7 +3,7 @@ from __future__ import absolute_import, unicode_literals from kombu.message import Message - +import base64 class BaseAsyncMessage(Message): """Base class for messages received on async client.""" @@ -17,10 +17,8 @@ class AsyncMessage(BaseAsyncMessage): """Serialized message.""" def encode(self, value): - return value - - def decode(self, value): - return value + """Encode/decode the value using Base64 encoding to avoid any illegal characters.""" + return base64.b64encode(value).decode('utf-8') def __getitem__(self, item): """Support Boto3-style access on a message.""" From e26deb72f9d15fd9799daaa6e8117364f03f280d Mon Sep 17 00:00:00 2001 From: Adam Szeptycki Date: Thu, 16 Feb 2017 20:12:17 +0100 Subject: [PATCH 37/44] Fix lint --- kombu/async/aws/sqs/message.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kombu/async/aws/sqs/message.py b/kombu/async/aws/sqs/message.py index 893e4f369..b4b283311 100644 --- a/kombu/async/aws/sqs/message.py +++ b/kombu/async/aws/sqs/message.py @@ -5,6 +5,7 @@ from kombu.message import Message import base64 + class BaseAsyncMessage(Message): """Base class for messages received on async client.""" @@ -15,11 +16,11 @@ class AsyncRawMessage(BaseAsyncMessage): class AsyncMessage(BaseAsyncMessage): """Serialized message.""" - + def encode(self, value): - """Encode/decode the value using Base64 encoding to avoid any illegal characters.""" + """Encode/decode the value using Base64 encoding.""" return base64.b64encode(value).decode('utf-8') - + def __getitem__(self, item): """Support Boto3-style access on a message.""" if item == 'ReceiptHandle': From 483f5bd5065cd22b6e230c870119cfbcf78f8c58 Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Thu, 16 Feb 2017 17:19:54 -0800 Subject: [PATCH 38/44] remove some unused things --- kombu/async/aws/connection.py | 10 ++-------- kombu/async/aws/ext.py | 3 +-- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index da3734a1d..dd7bbbd1d 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -11,11 +11,6 @@ import io -try: - from urllib.parse import urlunsplit -except ImportError: - from urlparse import urlunsplit # noqa - try: # pragma: no cover from email import message_from_bytes from email.mime.message import MIMEMessage @@ -87,7 +82,6 @@ class AsyncHTTPSConnection(object): method = 'GET' path = '/' body = None - scheme = 'https' default_ports = {'http': 80, 'https': 443} def __init__(self, strict=None, timeout=20.0, http_client=None, **kwargs): @@ -109,7 +103,7 @@ def request(self, method, path, body=None, headers=None): if headers is not None: self.headers.extend(list(items(headers))) - def getrequest(self, scheme=None): + def getrequest(self): headers = Headers(self.headers) return self.Request(self.path, method=self.method, headers=headers, body=self.body, connect_timeout=self.timeout, @@ -152,7 +146,7 @@ def __repr__(self): class AsyncConnection(object): """Async AWS Connection.""" - def __init__(self, sqs_connection, http_client=None, **kwargs): + def __init__(self, sqs_connection, http_client=None): self.sqs_connection = sqs_connection self._httpclient = http_client or get_client() diff --git a/kombu/async/aws/ext.py b/kombu/async/aws/ext.py index 4268073d3..3519ab655 100644 --- a/kombu/async/aws/ext.py +++ b/kombu/async/aws/ext.py @@ -5,11 +5,10 @@ try: import boto3 from botocore import exceptions - from boto3 import session from botocore.awsrequest import AWSRequest from botocore.response import get_response except ImportError: - boto3 = session = None + boto3 = None class _void(object): pass From c2d4e1a773bd9cfee1a925001dcb291cbb8694c1 Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Thu, 16 Feb 2017 17:45:18 -0800 Subject: [PATCH 39/44] removing unused stuff --- kombu/async/aws/connection.py | 8 ++++---- kombu/async/aws/sqs/connection.py | 6 +----- t/unit/async/aws/test_connection.py | 5 ----- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index dc3d85c4b..7c43cbf91 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -84,7 +84,7 @@ class AsyncHTTPSConnection(object): body = None default_ports = {'http': 80, 'https': 443} - def __init__(self, strict=None, timeout=20.0, http_client=None, **kwargs): + def __init__(self, strict=None, timeout=20.0, http_client=None): self.headers = [] self.timeout = timeout self.strict = strict @@ -123,7 +123,7 @@ def connect(self): def close(self): pass - def putrequest(self, method, path, **kwargs): + def putrequest(self, method, path): self.method = method self.path = path @@ -170,8 +170,8 @@ def _mexe(self, request, sender=None, callback=None): class AsyncAWSQueryConnection(AsyncConnection): """Async AWS Query Connection.""" - def __init__(self, sqs_connection, http_client=None, http_client_params={}, - **kwargs): + def __init__(self, sqs_connection, http_client=None, + http_client_params={}): AsyncConnection.__init__(self, sqs_connection, http_client, **http_client_params) diff --git a/kombu/async/aws/sqs/connection.py b/kombu/async/aws/sqs/connection.py index 02904afd4..13a9033ca 100644 --- a/kombu/async/aws/sqs/connection.py +++ b/kombu/async/aws/sqs/connection.py @@ -20,9 +20,7 @@ class AsyncSQSConnection(AsyncAWSQueryConnection): def __init__(self, sqs_connection, aws_access_key_id=None, aws_secret_access_key=None, - proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, debug=0, - https_connection_factory=None, region=None, *args, **kwargs): + debug=0, region=None, **kwargs): if boto3 is None: raise ImportError('boto3 is not installed') AsyncAWSQueryConnection.__init__( @@ -30,8 +28,6 @@ def __init__(self, sqs_connection, sqs_connection, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, - proxy=proxy, proxy_port=proxy_port, - proxy_user=proxy_user, proxy_pass=proxy_pass, region_name=region, debug=debug, https_connection_factory=https_connection_factory, **kwargs ) diff --git a/t/unit/async/aws/test_connection.py b/t/unit/async/aws/test_connection.py index 53b5e6d81..cba953b33 100644 --- a/t/unit/async/aws/test_connection.py +++ b/t/unit/async/aws/test_connection.py @@ -42,10 +42,6 @@ def side_effect(ret): class test_AsyncHTTPSConnection(AWSCase): - def test_AsyncHTTPSConnection(self): - x = AsyncHTTPSConnection() - assert x.scheme == 'https' - def test_http_client(self): x = AsyncHTTPSConnection() assert x.http_client is http.get_client() @@ -59,7 +55,6 @@ def test_args(self): ) assert x.strict assert x.timeout == 33.3 - assert x.scheme == 'https' def test_request(self): x = AsyncHTTPSConnection('aws.vandelay.com') From 6c093442c3e276a74a37033fffcc8a5cfc27bada Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Thu, 16 Feb 2017 17:54:14 -0800 Subject: [PATCH 40/44] ugh --- kombu/async/aws/connection.py | 6 +++--- kombu/async/aws/sqs/connection.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index 7c43cbf91..39e77a334 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -146,7 +146,7 @@ def __repr__(self): class AsyncConnection(object): """Async AWS Connection.""" - def __init__(self, sqs_connection, http_client=None): + def __init__(self, sqs_connection, http_client=None, **kwargs): self.sqs_connection = sqs_connection self._httpclient = http_client or get_client() @@ -171,9 +171,9 @@ class AsyncAWSQueryConnection(AsyncConnection): """Async AWS Query Connection.""" def __init__(self, sqs_connection, http_client=None, - http_client_params={}): + http_client_params={}, **kwargs): AsyncConnection.__init__(self, sqs_connection, http_client, - **http_client_params) + **http_client_params, **kwargs) def make_request(self, operation, params_, path, verb, callback=None): params = params_.copy() diff --git a/kombu/async/aws/sqs/connection.py b/kombu/async/aws/sqs/connection.py index 13a9033ca..64e9c5977 100644 --- a/kombu/async/aws/sqs/connection.py +++ b/kombu/async/aws/sqs/connection.py @@ -29,7 +29,7 @@ def __init__(self, sqs_connection, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=region, debug=debug, - https_connection_factory=https_connection_factory, **kwargs + **kwargs ) def create_queue(self, queue_name, From a738a008f8ccf028152df22009606809f4b7de84 Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Thu, 16 Feb 2017 19:43:55 -0800 Subject: [PATCH 41/44] ugh --- kombu/async/aws/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index 39e77a334..ad7df5218 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -173,7 +173,7 @@ class AsyncAWSQueryConnection(AsyncConnection): def __init__(self, sqs_connection, http_client=None, http_client_params={}, **kwargs): AsyncConnection.__init__(self, sqs_connection, http_client, - **http_client_params, **kwargs) + **http_client_params) def make_request(self, operation, params_, path, verb, callback=None): params = params_.copy() From ec7d0bfc2ad22ea74cea3ce01b5944405fa889e4 Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Thu, 16 Feb 2017 20:22:21 -0800 Subject: [PATCH 42/44] ugh --- kombu/async/aws/connection.py | 4 +++- kombu/async/aws/sqs/connection.py | 7 +------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index ad7df5218..d25db5c5d 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -171,7 +171,9 @@ class AsyncAWSQueryConnection(AsyncConnection): """Async AWS Query Connection.""" def __init__(self, sqs_connection, http_client=None, - http_client_params={}, **kwargs): + http_client_params=None, **kwargs): + if not http_client_params: + http_client_params = {} AsyncConnection.__init__(self, sqs_connection, http_client, **http_client_params) diff --git a/kombu/async/aws/sqs/connection.py b/kombu/async/aws/sqs/connection.py index 64e9c5977..7b0b738e0 100644 --- a/kombu/async/aws/sqs/connection.py +++ b/kombu/async/aws/sqs/connection.py @@ -17,17 +17,12 @@ class AsyncSQSConnection(AsyncAWSQueryConnection): """Async SQS Connection.""" - def __init__(self, sqs_connection, - aws_access_key_id=None, - aws_secret_access_key=None, - debug=0, region=None, **kwargs): + def __init__(self, sqs_connection, debug=0, region=None, **kwargs): if boto3 is None: raise ImportError('boto3 is not installed') AsyncAWSQueryConnection.__init__( self, sqs_connection, - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, region_name=region, debug=debug, **kwargs ) From 1659c6cf737c0e663b8c5c270da1b316688af7bc Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Thu, 16 Feb 2017 20:43:18 -0800 Subject: [PATCH 43/44] landscape ignoring --- kombu/async/aws/connection.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index d25db5c5d..c2c9ee7ae 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -146,7 +146,7 @@ def __repr__(self): class AsyncConnection(object): """Async AWS Connection.""" - def __init__(self, sqs_connection, http_client=None, **kwargs): + def __init__(self, sqs_connection, http_client=None, **kwargs): # noqa self.sqs_connection = sqs_connection self._httpclient = http_client or get_client() @@ -177,11 +177,11 @@ def __init__(self, sqs_connection, http_client=None, AsyncConnection.__init__(self, sqs_connection, http_client, **http_client_params) - def make_request(self, operation, params_, path, verb, callback=None): + def make_request(self, operation, params_, path, verb, callback=None): # noqa params = params_.copy() if operation: params['Action'] = operation - signer = self.sqs_connection._request_signer + signer = self.sqs_connection._request_signer # noqa # defaults for non-get signing_type = 'standard' @@ -201,7 +201,7 @@ def make_request(self, operation, params_, path, verb, callback=None): return self._mexe(prepared_request, callback=callback) def get_list(self, operation, params, markers, - path='/', parent=None, verb='POST', callback=None): + path='/', parent=None, verb='POST', callback=None): # noqa return self.make_request( operation, params, path, verb, callback=transform( @@ -211,7 +211,7 @@ def get_list(self, operation, params, markers, ) def get_object(self, operation, params, - path='/', parent=None, verb='GET', callback=None): + path='/', parent=None, verb='GET', callback=None): # noqa return self.make_request( operation, params, path, verb, callback=transform( @@ -220,7 +220,7 @@ def get_object(self, operation, params, ) def get_status(self, operation, params, - path='/', parent=None, verb='GET', callback=None): + path='/', parent=None, verb='GET', callback=None): # noqa return self.make_request( operation, params, path, verb, callback=transform( @@ -228,30 +228,30 @@ def get_status(self, operation, params, ), ) - def _on_list_ready(self, parent, markers, operation, response): + def _on_list_ready(self, parent, markers, operation, response): # noqa service_model = self.sqs_connection.meta.service_model if response.status == 200: - httpres, parsed = get_response( + _, parsed = get_response( service_model.operation_model(operation), response.response ) return parsed else: raise self._for_status(response, response.read()) - def _on_obj_ready(self, parent, operation, response): + def _on_obj_ready(self, parent, operation, response): # noqa service_model = self.sqs_connection.meta.service_model if response.status == 200: - httpres, parsed = get_response( + _, parsed = get_response( service_model.operation_model(operation), response.response ) return parsed else: raise self._for_status(response, response.read()) - def _on_status_ready(self, parent, operation, response): + def _on_status_ready(self, parent, operation, response): # noqa service_model = self.sqs_connection.meta.service_model if response.status == 200: - httpres, parsed = get_response( + httpres, _ = get_response( service_model.operation_model(operation), response.response ) return httpres.code From 06b56a4bdc3e1df2f372b691dd8df5d9c7343915 Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Thu, 16 Feb 2017 21:20:37 -0800 Subject: [PATCH 44/44] shut up, landscape --- examples/hello_consumer.py | 2 +- kombu/async/aws/connection.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/examples/hello_consumer.py b/examples/hello_consumer.py index 5722450e3..71f40f139 100644 --- a/examples/hello_consumer.py +++ b/examples/hello_consumer.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, unicode_literals, print_function -from kombu import Connection +from kombu import Connection # noqa with Connection('amqp://guest:guest@localhost:5672//') as conn: diff --git a/kombu/async/aws/connection.py b/kombu/async/aws/connection.py index c2c9ee7ae..303f35773 100644 --- a/kombu/async/aws/connection.py +++ b/kombu/async/aws/connection.py @@ -200,8 +200,7 @@ def make_request(self, operation, params_, path, verb, callback=None): # noqa # print(prepared_request.body) return self._mexe(prepared_request, callback=callback) - def get_list(self, operation, params, markers, - path='/', parent=None, verb='POST', callback=None): # noqa + def get_list(self, operation, params, markers, path='/', parent=None, verb='POST', callback=None): # noqa return self.make_request( operation, params, path, verb, callback=transform( @@ -210,8 +209,7 @@ def get_list(self, operation, params, markers, ), ) - def get_object(self, operation, params, - path='/', parent=None, verb='GET', callback=None): # noqa + def get_object(self, operation, params, path='/', parent=None, verb='GET', callback=None): # noqa return self.make_request( operation, params, path, verb, callback=transform( @@ -219,8 +217,7 @@ def get_object(self, operation, params, ), ) - def get_status(self, operation, params, - path='/', parent=None, verb='GET', callback=None): # noqa + def get_status(self, operation, params, path='/', parent=None, verb='GET', callback=None): # noqa return self.make_request( operation, params, path, verb, callback=transform(