From 8e17aeb414a3acb0ffc44931d430d51303f3482f Mon Sep 17 00:00:00 2001 From: Bo Lopker Date: Fri, 27 Nov 2015 08:42:14 -0800 Subject: [PATCH] FIX #651 try next Sentinel node on TimeoutError --- redis/sentinel.py | 7 ++++--- tests/test_sentinel.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/redis/sentinel.py b/redis/sentinel.py index 3fb89ce2e1..518fec582f 100644 --- a/redis/sentinel.py +++ b/redis/sentinel.py @@ -4,7 +4,8 @@ from redis.client import StrictRedis from redis.connection import ConnectionPool, Connection -from redis.exceptions import ConnectionError, ResponseError, ReadOnlyError +from redis.exceptions import (ConnectionError, ResponseError, ReadOnlyError, + TimeoutError) from redis._compat import iteritems, nativestr, xrange @@ -211,7 +212,7 @@ def discover_master(self, service_name): for sentinel_no, sentinel in enumerate(self.sentinels): try: masters = sentinel.sentinel_masters() - except ConnectionError: + except (ConnectionError, TimeoutError): continue state = masters.get(service_name) if state and self.check_master_state(state, service_name): @@ -235,7 +236,7 @@ def discover_slaves(self, service_name): for sentinel in self.sentinels: try: slaves = sentinel.sentinel_slaves(service_name) - except (ConnectionError, ResponseError): + except (ConnectionError, ResponseError, TimeoutError): continue slaves = self.filter_slaves(slaves) if slaves: diff --git a/tests/test_sentinel.py b/tests/test_sentinel.py index 0a6e98b273..220cbcf1f6 100644 --- a/tests/test_sentinel.py +++ b/tests/test_sentinel.py @@ -15,10 +15,12 @@ def __init__(self, cluster, id): def sentinel_masters(self): self.cluster.connection_error_if_down(self) + self.cluster.timeout_if_down(self) return {self.cluster.service_name: self.cluster.master} def sentinel_slaves(self, master_name): self.cluster.connection_error_if_down(self) + self.cluster.timeout_if_down(self) if master_name != self.cluster.service_name: return [] return self.cluster.slaves @@ -38,11 +40,16 @@ def __init__(self, service_name='mymaster', ip='127.0.0.1', port=6379): self.service_name = service_name self.slaves = [] self.nodes_down = set() + self.nodes_timeout = set() def connection_error_if_down(self, node): if node.id in self.nodes_down: raise exceptions.ConnectionError + def timeout_if_down(self, node): + if node.id in self.nodes_timeout: + raise exceptions.TimeoutError + def client(self, host, port, **kwargs): return SentinelTestClient(self, (host, port)) @@ -82,6 +89,15 @@ def test_discover_master_sentinel_down(cluster, sentinel): assert sentinel.sentinels[0].id == ('bar', 26379) +def test_discover_master_sentinel_timeout(cluster, sentinel): + # Put first sentinel 'foo' down + cluster.nodes_timeout.add(('foo', 26379)) + address = sentinel.discover_master('mymaster') + assert address == ('127.0.0.1', 6379) + # 'bar' is now first sentinel + assert sentinel.sentinels[0].id == ('bar', 26379) + + def test_master_min_other_sentinels(cluster): sentinel = Sentinel([('foo', 26379)], min_other_sentinels=1) # min_other_sentinels @@ -130,6 +146,12 @@ def test_discover_slaves(cluster, sentinel): cluster.nodes_down.add(('foo', 26379)) assert sentinel.discover_slaves('mymaster') == [ ('slave0', 1234), ('slave1', 1234)] + cluster.nodes_down.clear() + + # node0 -> TIMEOUT + cluster.nodes_timeout.add(('foo', 26379)) + assert sentinel.discover_slaves('mymaster') == [ + ('slave0', 1234), ('slave1', 1234)] def test_master_for(cluster, sentinel):