diff --git a/.travis.yml b/.travis.yml index 2619b406d..46f387a36 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,2 +1,8 @@ language: c script: bash ./travis.sh + +addons: + apt: + packages: + - python3 + - python3-pip diff --git a/conf/a_dc1.yml b/conf/a_dc1.yml deleted file mode 100644 index 74d043a64..000000000 --- a/conf/a_dc1.yml +++ /dev/null @@ -1,20 +0,0 @@ -dyn_o_mite: - datacenter: dc1 - rack: rack1 - dyn_listen: 127.0.0.1:8101 - dyn_seeds: - - 127.0.0.2:8101:rack2:dc2:1383429731 - - 127.0.0.3:8101:rack2:dc2:12345678 - - 127.0.0.4:8101:rack1:dc2:1383429731 - - 127.0.0.5:8101:rack1:dc2:12345678 - listen: 127.0.0.1:8102 - servers: - - 127.0.0.1:22121:1 - tokens: '1383429731' - secure_server_option: datacenter - pem_key_file: conf/dynomite.pem - data_store: 0 - stats_listen: 0.0.0.0:22221 - datastore_connections: 3 - local_peer_connections: 3 - remote_peer_connections: 3 diff --git a/conf/a_dc2_rack1_node1.yml b/conf/a_dc2_rack1_node1.yml deleted file mode 100644 index 0b1edb3fb..000000000 --- a/conf/a_dc2_rack1_node1.yml +++ /dev/null @@ -1,21 +0,0 @@ -dyn_o_mite: - datacenter: dc2 - rack: rack1 - dyn_listen: 127.0.0.2:8101 - dyn_seeds: - - 127.0.0.1:8101:rack1:dc1:1383429731 - - 127.0.0.4:8101:rack2:dc2:1383429731 - - 127.0.0.5:8101:rack2:dc2:12345678 - - 127.0.0.3:8101:rack1:dc2:12345678 - listen: 127.0.0.2:8102 - servers: - - 127.0.0.1:22122:1 - tokens: '1383429731' - secure_server_option: datacenter - pem_key_file: conf/dynomite.pem - data_store: 0 - stats_listen: 0.0.0.0:22222 - datastore_connections: 3 - local_peer_connections: 3 - remote_peer_connections: 3 - diff --git a/conf/a_dc2_rack1_node2.yml b/conf/a_dc2_rack1_node2.yml deleted file mode 100644 index b7aa5bb31..000000000 --- a/conf/a_dc2_rack1_node2.yml +++ /dev/null @@ -1,22 +0,0 @@ -dyn_o_mite: - datacenter: dc2 - rack: rack1 - dyn_listen: 127.0.0.3:8101 - dyn_seeds: - - 127.0.0.1:8101:rack1:dc1:1383429731 - - 127.0.0.4:8101:rack2:dc2:1383429731 - - 127.0.0.5:8101:rack2:dc2:12345678 - - 127.0.0.2:8101:rack1:dc2:1383429731 - listen: 127.0.0.3:8102 - servers: - - 127.0.0.1:22123:1 - tokens: '12345678' - secure_server_option: datacenter - pem_key_file: conf/dynomite.pem - data_store: 0 - stats_listen: 0.0.0.0:22223 - datastore_connections: 1 - local_peer_connections: 2 - remote_peer_connections: 2 - - diff --git a/conf/a_dc2_rack2_node1.yml b/conf/a_dc2_rack2_node1.yml deleted file mode 100644 index 5e0b5ff55..000000000 --- a/conf/a_dc2_rack2_node1.yml +++ /dev/null @@ -1,20 +0,0 @@ -dyn_o_mite: - datacenter: dc2 - rack: rack2 - dyn_listen: 127.0.0.4:8101 - dyn_seeds: - - 127.0.0.1:8101:rack1:dc1:1383429731 - - 127.0.0.5:8101:rack2:dc2:12345678 - - 127.0.0.2:8101:rack1:dc2:1383429731 - - 127.0.0.3:8101:rack1:dc2:12345678 - listen: 127.0.0.4:8102 - servers: - - 127.0.0.1:22124:1 - tokens: '1383429731' - secure_server_option: datacenter - pem_key_file: conf/dynomite.pem - data_store: 0 - stats_listen: 0.0.0.0:22224 - enable_gossip: false - mbuf_size: 16384 - max_msgs: 300000 diff --git a/conf/a_dc2_rack2_node2.yml b/conf/a_dc2_rack2_node2.yml deleted file mode 100644 index c703b73c4..000000000 --- a/conf/a_dc2_rack2_node2.yml +++ /dev/null @@ -1,21 +0,0 @@ -dyn_o_mite: - datacenter: dc2 - rack: rack2 - dyn_listen: 127.0.0.5:8101 - dyn_seeds: - - 127.0.0.1:8101:rack1:dc1:1383429731 - - 127.0.0.4:8101:rack2:dc2:1383429731 - - 127.0.0.2:8101:rack1:dc2:1383429731 - - 127.0.0.3:8101:rack1:dc2:12345678 - listen: 127.0.0.5:8102 - servers: - - 127.0.0.1:22125:1 - tokens: '12345678' - secure_server_option: datacenter - pem_key_file: conf/dynomite.pem - data_store: 0 - stats_listen: 0.0.0.0:22225 - datastore_connections: 3 - local_peer_connections: 3 - remote_peer_connections: 3 - diff --git a/scripts/dynomite/dyn_mc_test.py b/scripts/dynomite/dyn_mc_test.py index 45fb5ab7e..498432219 100644 --- a/scripts/dynomite/dyn_mc_test.py +++ b/scripts/dynomite/dyn_mc_test.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from optparse import OptionParser -import ConfigParser +import configparser import logging import time import os @@ -62,7 +62,7 @@ def main(): help="Number of keys\n") if len(sys.argv) == 1: - print "Learn some usages: " + sys.argv[0] + " -h" + print("Learn some usages: " + sys.argv[0] + " -h") sys.exit(1) @@ -78,7 +78,7 @@ def main(): #fh.setFormatter(formatter) #logger.addHandler(fh) - print options + print(options) logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s', @@ -91,7 +91,7 @@ def main(): numkeys = int(options.numkeys) start = int(options.skipkeys) end = int(options.numkeys) - print 'start: ' + str(start) + ' and end: ' + str(end) + print('start: ' + str(start) + ' and end: ' + str(end)) if 'write' == options.operation : for i in range(start, end ) : @@ -103,14 +103,14 @@ def main(): value = mc.get('key_' + str(i)) if value is None: error_count = error_count + 1 - print 'No value for key: ' + 'key_' + str(i) + print('No value for key: ' + 'key_' + str(i)) else : - print 'key_' + str(i) + ' has value : ' + value - print 'Errour count: ' + str(error_count) + print('key_' + str(i) + ' has value : ' + value) + print('Errour count: ' + str(error_count)) elif 'mread' == options.operation : n = (end - start) / 10 n = min(n, 10) - print n + print(n) keys = [] i = 0 while (i < n) : @@ -119,13 +119,13 @@ def main(): if key not in keys : keys.append(key) i = i + 1 - print keys + print(keys) #values = mc.get_multi(['key_1', 'key_2', 'key_3']) while (len(keys) > 0) : values = mc.get_multi(keys) - print values - for key in values.keys() : + print(values) + for key in values.keys(): keys.remove(key) @@ -143,7 +143,7 @@ def main(): if value != None : is_stop = True - print 'Estimated elapsed time : ' + str(current_milli_time() - int(value)) + print('Estimated elapsed time : ' + str(current_milli_time() - int(value))) elif 'sdel' == options.operation : mc.delete('key_time') diff --git a/scripts/dynomite/dyn_redis_purge.py b/scripts/dynomite/dyn_redis_purge.py index 834519467..68174482f 100644 --- a/scripts/dynomite/dyn_redis_purge.py +++ b/scripts/dynomite/dyn_redis_purge.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from optparse import OptionParser -import ConfigParser +import configparser import logging import time import os @@ -40,7 +40,7 @@ def run(self): host = self.options.host port = self.options.port - print "Starting thread: " + self.name + ", filename: " + self.filename + print("Starting thread: " + self.name + ", filename: " + self.filename) # Get lock to synchronize threads #threadLock.acquire() @@ -69,7 +69,7 @@ def rebalance_ops(filename, host, port, db): i = 0 for line in open(filename,'r').readlines(): if line != '': - print line + print(line) line = line.strip('\n') if line == '': continue @@ -81,7 +81,7 @@ def rebalance_ops(filename, host, port, db): if (i % 5000 == 0): time.sleep(1) except redis.exceptions.ResponseError: - print "reconnecting ..." + print("reconnecting ...") r1 = redis.StrictRedis(host, port, db=0) @@ -123,13 +123,13 @@ def main(): if len(sys.argv) == 1: - print "Learn some usages: " + sys.argv[0] + " -h" + print("Learn some usages: " + sys.argv[0] + " -h") sys.exit(1) (options, args) = parser.parse_args() - print options + print(options) num_threads = int(options.th) @@ -147,7 +147,7 @@ def main(): for t in threads: t.join() - print "" + print() if __name__ == '__main__': diff --git a/scripts/dynomite/dyn_redis_test.py b/scripts/dynomite/dyn_redis_test.py index 13b92ddb6..1f0abcb2b 100755 --- a/scripts/dynomite/dyn_redis_test.py +++ b/scripts/dynomite/dyn_redis_test.py @@ -1,11 +1,11 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 #Requirement: # Install https://github.com/andymccurdy/redis-py from optparse import OptionParser -import ConfigParser +import configparser import logging import time import os @@ -48,7 +48,7 @@ def run(self): delay = self.options.delay start = self.start_num end = self.end_num - print "Starting thread: " + self.name + ", start: " + str(start) + " and end: " + str(end) + print("Starting thread: " + self.name + ", start: " + str(start) + " and end: " + str(end)) # Get lock to synchronize threads #threadLock.acquire() @@ -77,7 +77,7 @@ def run(self): if value != None : is_stop = True - print 'Estimated elapsed time : ' + str(current_milli_time() - int(value)) + print('Estimated elapsed time : ' + str(current_milli_time() - int(value))) elif 'sdel' == operation : r = redis.StrictRedis(host, port, db=0) r.delete('key_time') @@ -103,8 +103,8 @@ def write_ops(skipkeys, numkeys, delay, host, port, db): conns = get_conns(host, port, db, num_conn) start = int(skipkeys) end = int(numkeys) - print 'start: ' + str(start) + ' and end: ' + str(end) - print 'payload_prefix: ' + payload_prefix + print('start: ' + str(start) + ' and end: ' + str(end)) + print('payload_prefix: ' + payload_prefix) for i in range(start, end ) : r = conns[i % num_conn] @@ -114,7 +114,7 @@ def write_ops(skipkeys, numkeys, delay, host, port, db): r.set('key_' + str(i), generate_value(i)) time.sleep(int(delay)) except redis.exceptions.ResponseError: - print "reconnecting ..." + print("reconnecting ...") r = redis.StrictRedis(host, port, db=0) conns[i % num_conn] = r @@ -125,7 +125,7 @@ def read_ops(skipkeys, numkeys, delay, host, port, db): start = int(skipkeys) end = int(numkeys) - print 'start: ' + str(start) + ' and end: ' + str(end) + print('start: ' + str(start) + ' and end: ' + str(end)) error_count = 0 for i in range(start, end ) : r = conns[i % num_conn] @@ -133,17 +133,17 @@ def read_ops(skipkeys, numkeys, delay, host, port, db): value = r.get('key_' + str(i)) time.sleep(int(delay)) except redis.exceptions.ResponseError: - print "reconnecting ..." + print("reconnecting ...") r = redis.StrictRedis(host, port, db=0) if value is None: error_count = error_count + 1 - print 'No value for key: ' + 'key_' + str(i) + print('No value for key: ' + 'key_' + str(i)) elif value != generate_value(i): - print 'key_' + str(i) + ' has incorrect value ' + print('key_' + str(i) + ' has incorrect value ') error_count += 1 - print 'Error count: ' + str(error_count) + print('Error count: ' + str(error_count)) def del_ops(skipkeys, numkeys, delay, host, port, db): @@ -152,7 +152,7 @@ def del_ops(skipkeys, numkeys, delay, host, port, db): start = int(skipkeys) end = int(numkeys) - print 'start: ' + str(start) + ' and end: ' + str(end) + print('start: ' + str(start) + ' and end: ' + str(end)) for i in range(start, end ) : r = conns[i % num_conn] @@ -163,7 +163,7 @@ def del_ops(skipkeys, numkeys, delay, host, port, db): r.delete('key_' + str(i)) time.sleep(int(delay)) except redis.exceptions.ResponseError: - print "reconnecting ..." + print("reconnecting ...") r = redis.StrictRedis(host, port, db=0) @@ -172,11 +172,11 @@ def mread_ops(skipkeys, numkeys, delay, host, port, db): start = int(skipkeys) end = int(numkeys) - print 'start: ' + str(start) + ' and end: ' + str(end) + print('start: ' + str(start) + ' and end: ' + str(end)) n = (end - start) / 10 n = min(n, 10) - print n + print(n) keys = [] i = 0 while (i < n) : @@ -185,13 +185,13 @@ def mread_ops(skipkeys, numkeys, delay, host, port, db): if key not in keys : keys.append(key) i = i + 1 - print keys + print(keys) while (len(keys) > 0) : values = r.mget(keys) - print values + print(values) time.sleep(int(delay)) - for key in values.keys() : + for key in values.keys(): keys.remove(key) @@ -247,13 +247,13 @@ def main(): if len(sys.argv) == 1: - print "Learn some usages: " + sys.argv[0] + " -h" + print("Learn some usages: " + sys.argv[0] + " -h") sys.exit(1) (options, args) = parser.parse_args() - print options + print(options) start = int(options.skipkeys) end = int(options.numkeys) global payload_prefix @@ -263,7 +263,7 @@ def main(): step = (end - start) / num_threads - print "step " + str(step) + print("step " + str(step)) for i in range(0, num_threads): if (i != num_threads-1): @@ -278,7 +278,7 @@ def main(): for t in threads: t.join() - print "" + print() if __name__ == '__main__': diff --git a/scripts/dynomite/generate_yamls.py b/scripts/dynomite/generate_yamls.py index dcc314b86..78a49dfc7 100755 --- a/scripts/dynomite/generate_yamls.py +++ b/scripts/dynomite/generate_yamls.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 ''' script for generating dynomite yaml files for every node in a cluster. diff --git a/scripts/redis/redis-check.py b/scripts/redis/redis-check.py index 284b83aa5..8a402683b 100644 --- a/scripts/redis/redis-check.py +++ b/scripts/redis/redis-check.py @@ -7,17 +7,17 @@ r = redis.StrictRedis(host='localhost', port=port, db=0) # lrange -print [r.lrange('lfoo', 0, x) for x in xrange(1, range)] -print [r.lpush('lfoo', str(x)*factor) for x in xrange(1, range)] -print [r.lrange('lfoo', 0, x) for x in xrange(1, range)] -print r.delete('lfoo') +print([r.lrange('lfoo', 0, x) for x in range(1, range)]) +print([r.lpush('lfoo', str(x)*factor) for x in range(1, range)]) +print([r.lrange('lfoo', 0, x) for x in range(1, range)]) +print(r.delete('lfoo')) # del -print [r.set('foo' + str(x), str(x)*factor) for x in xrange(1, range)] -keys = ['foo' + str(x) for x in xrange(1, range)] -print [r.delete(keys) for x in xrange(1, range)] +print([r.set('foo' + str(x), str(x)*factor) for x in range(1, range)]) +keys = ['foo' + str(x) for x in range(1, range)] +print([r.delete(keys) for x in range(1, range)]) # mget -print [r.set('foo' + str(x), str(x)*100) for x in xrange(1, range)] -keys = ['foo' + str(x) for x in xrange(1, range)] -print [r.mget(keys) for x in xrange(1, range)] +print([r.set('foo' + str(x), str(x)*100) for x in range(1, range)]) +keys = ['foo' + str(x) for x in range(1, range)] +print([r.mget(keys) for x in range(1, range)]) diff --git a/test/cluster_generator.py b/test/cluster_generator.py new file mode 100755 index 000000000..2dbe5d5f4 --- /dev/null +++ b/test/cluster_generator.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +from collections import namedtuple +from contextlib import ExitStack +from contextlib import contextmanager +from signal import SIGINT +from tempfile import mkdtemp +from time import sleep +from urllib.request import urlopen +import argparse +import json +import random +import socket +import sys +import yaml + +from plumbum import FG +from plumbum import BG +from plumbum import LocalPath +from plumbum import local + +from ip_util import quad2int +from ip_util import int2quad +from dyno_cluster import DynoCluster +from dyno_node import DynoNode +from func_test import comparison_test +from redis_node import RedisNode + +DYN_O_MITE_DEFAULTS = dict( + secure_server_option='datacenter', + pem_key_file='conf/dynomite.pem', + data_store=0, + datastore_connections=1, +) +INTERNODE_LISTEN = 8101 +CLIENT_LISTEN = 8102 +REDIS_PORT = 1212 +STATS_PORT = 22222 +BASE_IPADDRESS = quad2int('127.0.1.1') +RING_SIZE = 2**32 + +SETTLE_TIME = 5 + +redis = local.get('./test/_binaries/redis-server', 'redis-server') +dynomite = local.get('./test/_binaries/dynomite', 'src/dynomite') + +@contextmanager +def launch_redis(ip): + logfile = 'logs/redis_{}.log'.format(ip) + f = (redis['--bind', ip, '--port', REDIS_PORT] > logfile) & BG(-9) + try: + yield RedisNode(ip, REDIS_PORT) + finally: + f.proc.kill() + f.wait() + +def pick_tokens(count, start_offset): + stride = RING_SIZE // count + token = start_offset + for i in range(count): + yield token % RING_SIZE + token += stride + +def tokens_for_rack(count): + offset = random.randrange(0, RING_SIZE) + return list(pick_tokens(count, offset)) + +def tokens_for_dc(racks): + return [ + (name, tokens_for_rack(count)) + for name, count in racks + ] + +def tokens_for_cluster(dcs, seed): + if seed is not None: + random.seed(seed) + + return [ + (dc['name'], tokens_for_dc(dc['racks'])) + for dc in dcs + ] + + +def dc_count(dc): + return sum(count for rack, count in dc) + +def generate_ips(): + state = BASE_IPADDRESS + while True: + yield int2quad(state) + state += 1 + +class DynoSpec(namedtuple('DynoSpec', 'ip port dc rack token ' + 'local_connections remote_connections seed_string')): + """Specifies how to launch a dynomite node""" + + def __new__(cls, ip, port, rack, dc, token, local_connections, + remote_connections): + seed_string = '{}:{}:{}:{}:{}'.format(ip, port, rack, dc, token) + return super(DynoSpec, cls).__new__(cls, ip, port, rack, dc, token, + local_connections, remote_connections, seed_string) + + def _generate_config(self, seeds_list): + conf = dict(DYN_O_MITE_DEFAULTS) + conf['datacenter'] = self.dc + conf['rack'] = self.rack + dyn_listen = '{}:{}'.format(self.ip, self.port) + conf['dyn_listen'] = dyn_listen + conf['listen'] = '{}:{}'.format(self.ip, CLIENT_LISTEN) + + # filter out our own seed string + conf['dyn_seeds'] = [s for s in seeds_list if s != self.seed_string] + conf['servers'] = ['{}:{}:0'.format(self.ip, REDIS_PORT)] + conf['stats_listen'] = '{}:{}'.format(self.ip, STATS_PORT) + conf['tokens'] = self.token + conf['local_peer_connections'] = self.local_connections + conf['remote_peer_connections'] = self.remote_connections + return dict(dyn_o_mite=conf) + + def write_config(self, seeds_list): + config = self._generate_config(seeds_list) + filename = 'conf/{}:{}:{}.yml'.format(self.dc, self.rack, self.token) + with open(filename, 'w') as fh: + yaml.dump(config, fh, default_flow_style=False) + return filename + + @contextmanager + def launch(self, seeds_list): + config_filename = self.write_config(seeds_list) + + with launch_redis(self.ip): + logfile = 'logs/dynomite_{}.log'.format(self.ip) + dynomite_future = dynomite['-o', logfile, '-c', config_filename, + '-v6'] & BG(-9) + + try: + yield DynoNode(self.ip) + finally: + dynomite_future.proc.kill() + dynomite_future.wait() + + +@contextmanager +def launch_dynomite(nodes): + seeds_list = [n.seed_string for n in nodes] + launched_nodes = [DynoNode(n.ip) for n in nodes] + with ExitStack() as stack: + launched_nodes = [ + stack.enter_context(n.launch(seeds_list)) + for n in nodes + ] + + yield DynoCluster(launched_nodes) + +def dict_request(request): + """Converts the request into an easy to consume dict format. + + We don't ingest the request in this format originally because dict + ordering is nondeterministic, and one of our goals is to deterministically + generate clusters.""" + return dict( + (dc['name'], dict(dc['racks'])) + for dc in request + ) + +def sum_racks(dcs): + return dict( + (name, sum(racks.values())) + for name, racks in dcs.items() + ) + +def generate_nodes(request, ips): + tokens = tokens_for_cluster(request, None) + counts_by_rack = dict_request(request) + counts_by_dc = sum_racks(counts_by_rack) + total_nodes = sum(counts_by_dc.values()) + for dc, racks in tokens: + dc_count = counts_by_dc[dc] + rack_count = counts_by_rack[dc] + remote_count = total_nodes - dc_count + for rack, tokens in racks: + local_count = rack_count[rack] - 1 + for token in tokens: + ip = next(ips) + yield DynoSpec(ip, INTERNODE_LISTEN, dc, rack, token, + local_count, remote_count) + +def setup_temp_dir(): + tempdir = LocalPath(mkdtemp(dir='.', prefix='test_run.')) + (tempdir / 'logs').mkdir() + confdir = (tempdir / 'conf') + confdir.mkdir() + + LocalPath('../../conf/dynomite.pem').symlink(confdir / 'dynomite.pem') + + return tempdir + +def main(): + parser = argparse.ArgumentParser( + description='Autogenerates a Dynomite cluster and runs functional ' + + 'tests against it') + parser.add_argument('request_file', default='test/request.yaml', + help='YAML file describing desired cluster', nargs='?') + args = parser.parse_args() + + with open(args.request_file, 'r') as fh: + request = yaml.load(fh) + + temp = setup_temp_dir() + + ips = generate_ips() + standalone_redis_ip = next(ips) + nodes = list(generate_nodes(request, ips)) + + with ExitStack() as stack: + with local.cwd(temp): + redis_info = stack.enter_context(launch_redis(standalone_redis_ip)) + dynomite_info = stack.enter_context(launch_dynomite(nodes)) + + sleep(SETTLE_TIME) + comparison_test(redis_info, dynomite_info, False) + + random_node = random.choice(dynomite_info.nodes) + stats_url = 'http://{}:{}/info'.format(random_node.ip, STATS_PORT) + json.loads(urlopen(stats_url).read().decode('ascii')) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/test/dual_run.py b/test/dual_run.py index b6cf51ad9..5250a9203 100644 --- a/test/dual_run.py +++ b/test/dual_run.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import redis class ResultMismatchError(Exception): @@ -33,19 +33,19 @@ def run_verify(self, func, *args): try: d_result = d_func(*args) if i > 0: - print "\tSucceeded in attempt {}".format(i+1) + print("\tSucceeded in attempt {}".format(i+1)) break - except redis.exceptions.ResponseError, e: + except redis.exceptions.ResponseError as e: if "Peer Node is not connected" in str(e): i = i + 1 - print "\tGot error '{}' ... Retry effort {}/{}\n\tQuery '{} {}'".format(e, i, retry_limit, func, str(args)) + print("\tGot error '{}' ... Retry effort {}/{}\n\tQuery '{} {}'".format(e, i, retry_limit, func, str(args))) continue - print "\tGot error '{}'\n\tQuery '{} {}'".format(e, func, str(args)) + print("\tGot error '{}'\n\tQuery '{} {}'".format(e, func, str(args))) break if self.debug: - print "Query: %s %s" % (func, str(args)) - print "Redis: %s" % str(r_result) - print "Dyno : %s" % str(d_result) + print("Query: %s %s" % (func, str(args))) + print("Redis: %s" % str(r_result)) + print("Dyno : %s" % str(d_result)) if r_result != d_result: raise ResultMismatchError(r_result, d_result, func, *args) return d_result diff --git a/test/dyno_cluster.py b/test/dyno_cluster.py index 237a3634f..8368db55d 100644 --- a/test/dyno_cluster.py +++ b/test/dyno_cluster.py @@ -1,11 +1,11 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import redis import random -#from dyno_node import DynoNode class DynoCluster(object): def __init__(self, nodes): - self.nodes = nodes + self.nodes = tuple(nodes) + def get_connection(self): node = random.choice(self.nodes) return node.get_connection() diff --git a/test/dyno_node.py b/test/dyno_node.py index 41605b5d8..3e792c460 100755 --- a/test/dyno_node.py +++ b/test/dyno_node.py @@ -1,19 +1,19 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import redis from node import Node from redis_node import RedisNode from dyno_cluster import DynoCluster class DynoNode(Node): - def __init__(self, host="localhost", ip="127.0.0.1", port=8102, - dnode_port=8101, data_store_port=22122): - super(DynoNode, self).__init__(host, ip, port) + def __init__(self, ip, port=8102, + dnode_port=8101, data_store_port=1212): + super(DynoNode, self).__init__(ip, port) self.name="Dyno" + self.name self.conf_file = None self.dnode_port = dnode_port - self.data_store_node = RedisNode(host, ip, data_store_port) + self.data_store_node = RedisNode(ip, data_store_port) def get_connection(self): # should return the connection to the dyno port not the redis - print "returning connection at %s:%d" % (self.host, self.port) - return redis.StrictRedis(self.host, self.port, db=0) + print("returning connection at %s:%d" % (self.ip, self.port)) + return redis.StrictRedis(self.ip, self.port, db=0) diff --git a/test/func_test.py b/test/func_test.py index 44e1861c0..7b949db44 100755 --- a/test/func_test.py +++ b/test/func_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import redis import argparse import random @@ -23,7 +23,7 @@ def create_key(test_name, key_id): def run_key_value_tests(c, max_keys=1000, max_payload=1024): #Set some test_name="KEY_VALUE" - print "Running %s tests" % test_name + print("Running %s tests" % test_name) for x in range(0, max_keys): key = create_key(test_name, x) c.run_verify("set", key, string_generator(size=random.randint(1, max_payload))) @@ -39,13 +39,13 @@ def run_key_value_tests(c, max_keys=1000, max_payload=1024): # expire a few key = create_key(test_name, random.randint(0, max_keys-1)) c.run_verify("expire", key, 5) - time.sleep(7); + time.sleep(7) c.run_verify("exists", key) def run_multikey_test(c, max_keys=1000, max_payload=10): #Set some test_name="MULTIKEY" - print "Running %s tests" % test_name + print("Running %s tests" % test_name) for n in range(0, 100): kv_pairs = {} len = random.randint(1, 50) @@ -69,12 +69,12 @@ def create_key_field(keyid=None, fieldid=None): keyid = random.randint(0, max_keys - 1) if fieldid is None: fieldid = random.randint(0, max_fields- 1) - key = create_key(test_name, keyid); - field = create_key("_field", fieldid); + key = create_key(test_name, keyid) + field = create_key("_field", fieldid) return (key, key + field) test_name="HASH_MAP" - print "Running %s tests" % test_name + print("Running %s tests" % test_name) #hset for key_iter in range(0, max_keys): @@ -130,30 +130,34 @@ def create_key_field(keyid=None, fieldid=None): #if next_index == 0: #break +def comparison_test(redis, dynomite, debug): + r_c = redis.get_connection() + d_c = dynomite.get_connection() + c = dual_run(r_c, d_c, debug) + run_key_value_tests(c) + + # XLarge payloads + run_key_value_tests(c, max_keys=10, max_payload=5*1024*1024) + run_multikey_test(c) + run_hash_tests(c, max_keys=10, max_fields=100) + print("All test ran fine") + def main(args): # This test assumes for now that the nodes are running at the given ports. # This is done by travis.sh. Please check that file and the corresponding # yml files for each dynomite instance there to get an idea of the topology. - r = RedisNode(host="localhost", ip="127.0.0.1", port=1212) - d1 = DynoNode(host="127.0.0.1", ip="127.0.0.1", data_store_port=22121) - d2 = DynoNode(host="127.0.0.2", ip="127.0.0.2", data_store_port=22122) - d3 = DynoNode(host="127.0.0.3", ip="127.0.0.3", data_store_port=22123) - d4 = DynoNode(host="127.0.0.4", ip="127.0.0.4", data_store_port=22124) - d5 = DynoNode(host="127.0.0.5", ip="127.0.0.5", data_store_port=22125) + r = RedisNode(ip="127.0.1.1", port=1212) + d1 = DynoNode(ip="127.0.1.2", data_store_port=22121) + d2 = DynoNode(ip="127.0.1.3", data_store_port=22122) + d3 = DynoNode(ip="127.0.1.4", data_store_port=22123) + d4 = DynoNode(ip="127.0.1.5", data_store_port=22124) + d5 = DynoNode(ip="127.0.1.6", data_store_port=22125) dyno_nodes = [d1,d2,d3,d4,d5] cluster = DynoCluster(dyno_nodes) - r_c = r.get_connection() - d_c = cluster.get_connection() - c = dual_run(r_c, d_c, args.debug) try: - run_key_value_tests(c) - # XLarge payloads - run_key_value_tests(c, max_keys=10, max_payload=5*1024*1024) - run_multikey_test(c) - run_hash_tests(c, max_keys=10, max_fields=100) - print "All test ran fine" + comparison_test(r, cluster, args.debug) except ResultMismatchError as r: - print r; + print(r) return 1 return 0 diff --git a/test/ip_util.py b/test/ip_util.py new file mode 100644 index 000000000..af926b58c --- /dev/null +++ b/test/ip_util.py @@ -0,0 +1,8 @@ +import socket +import struct + +def quad2int(ip): + return struct.unpack("!L", socket.inet_aton(ip))[0] + +def int2quad(ip): + return socket.inet_ntoa(struct.pack('!L', ip)) diff --git a/test/load.py b/test/load.py index 8590ba11f..d96a90375 100755 --- a/test/load.py +++ b/test/load.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 ## # A very simple basic program that will load data in a dynomite/redis server. diff --git a/test/node.py b/test/node.py index abca280a9..7ddd17c23 100755 --- a/test/node.py +++ b/test/node.py @@ -1,13 +1,16 @@ -#!/usr/bin/python +#!/usr/bin/env python3 class Node(object): - def __init__(self, host="localhost", ip="127.0.0.1", port=1212): - self.host=host + def __init__(self, ip, port): + self.ip = ip self.port=port - self.name="Node: %s:%d" % (host, port) + self.name="Node: %s:%d" % (ip, port) + def __name__(self): return self.name + def start(self): return + def stop(self): return diff --git a/test/redis_node.py b/test/redis_node.py index 2862e4d64..757fec414 100755 --- a/test/redis_node.py +++ b/test/redis_node.py @@ -1,12 +1,11 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import redis from node import Node class RedisNode(Node): - - def __init__(self, host, ip, port): - super(RedisNode, self).__init__(host, ip, port) + def __init__(self, ip, port): + super(RedisNode, self).__init__(ip, port) self.name = "Redis" + self.name def get_connection(self): - return redis.StrictRedis(self.host, self.port, db=0) + return redis.StrictRedis(self.ip, self.port, db=0) diff --git a/test/request.yaml b/test/request.yaml new file mode 100644 index 000000000..88a7f2167 --- /dev/null +++ b/test/request.yaml @@ -0,0 +1,8 @@ +- name: dc1 + racks: + - [rack1, 1] +- name: dc2 + racks: + - [rack1, 1] + - [rack2, 2] + - [rack3, 4] diff --git a/test/run_loop.py b/test/run_loop.py index d6253aeb1..014d9720e 100755 --- a/test/run_loop.py +++ b/test/run_loop.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import subprocess, time def run_command(cmd): @@ -16,14 +16,14 @@ def run_command(cmd): def main(): cmd = "bash travis.sh -n" for i in range(0, 50): - print"Running loop {}".format(i+1) + print("Running loop {}".format(i+1)) success, proc = run_command(cmd) if not success: for line in iter(proc.stderr.readline, b''): - print("--- " + line.rstrip()) + print(("--- " + line.rstrip())) break else: - print "...........................Success\n" + print("...........................Success\n") time.sleep(30) if __name__ == "__main__": diff --git a/test/supplemental.sh b/test/supplemental.sh new file mode 100755 index 000000000..adff51526 --- /dev/null +++ b/test/supplemental.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# check a single stats port +curl -s 127.0.1.2:22222/info | python -mjson.tool > /dev/null diff --git a/test/utils.py b/test/utils.py index 60930f679..0960a5d3b 100755 --- a/test/utils.py +++ b/test/utils.py @@ -1,8 +1,8 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import random import string -def string_generator(size=6, chars=string.letters + string.digits): +def string_generator(size=6, chars=string.ascii_letters + string.digits): return ''.join(random.choice(chars) for _ in range(size)) def number_generator(size=4, chars=string.digits): diff --git a/travis.sh b/travis.sh index ec3214584..cd41ef746 100755 --- a/travis.sh +++ b/travis.sh @@ -1,10 +1,7 @@ #!/bin/bash if [ -n "$TRAVIS" ]; then - - #python libs - sudo pip install redis - sudo pip install git+https://github.com/andymccurdy/redis-py.git@2.10.3 + sudo pip3 install redis plumbum pyyaml fi @@ -21,6 +18,10 @@ while [ "$#" -gt 0 ]; do esac done +set -o errexit +set -o nounset +set -o pipefail + #build Dynomite if [[ "${rebuild}" == "true" ]]; then @@ -29,104 +30,4 @@ else echo "not rebuilding Dynomite" fi -# Create the environment - -rm -rf test/_binaries/ -mkdir test/_binaries -rm -rf test/logs -mkdir test/logs -rm test/conf -ln -s ../conf test/conf -cp `pwd`/src/dynomite test/_binaries/ -cp `which redis-server` test/_binaries/ -cp `which redis-cli` test/_binaries/ -cd test - -# launch processes -function launch_redis() { - ./_binaries/redis-server --port 1212 > ./logs/redis_standalone.log & - ./_binaries/redis-server --port 22121 > ./logs/redis_22121.log & - ./_binaries/redis-server --port 22122 > ./logs/redis_22122.log & - ./_binaries/redis-server --port 22123 > ./logs/redis_22123.log & - ./_binaries/redis-server --port 22124 > ./logs/redis_22124.log & - ./_binaries/redis-server --port 22125 > ./logs/redis_22125.log & -} -function launch_dynomite() { - ./_binaries/dynomite -d -o ./logs/a_dc1.log \ - -c ./conf/a_dc1.yml -v6 - ./_binaries/dynomite -d -o ./logs/a_dc2_rack1_node1.log \ - -c ./conf/a_dc2_rack1_node1.yml -v6 - ./_binaries/dynomite -d -o ./logs/a_dc2_rack1_node2.log \ - -c ./conf/a_dc2_rack1_node2.yml -v6 - ./_binaries/dynomite -d -o ./logs/a_dc2_rack2_node1.log \ - -c ./conf/a_dc2_rack2_node1.yml -v6 - ./_binaries/dynomite -d -o ./logs/a_dc2_rack2_node2.log \ - -c ./conf/a_dc2_rack2_node2.yml -v6 -} - -function kill_redis() { - killall redis-server -} - -function kill_dynomite() { - killall dynomite -} - -declare -i RESULT -RESULT=0 - -function cleanup_and_exit() { - kill_redis - kill_dynomite - exit $RESULT -} - -launch_redis -launch_dynomite -DYNOMITE_NODES=`pgrep dynomite | wc -l` -REDIS_NODES=`pgrep redis-server | wc -l` - -if [[ $DYNOMITE_NODES -ne 5 ]]; then - echo "Not all dynomite nodes are running" >&2 - RESULT=1 - cleanup_and_exit -fi -if [[ $REDIS_NODES -ne 6 ]]; then - echo "Not all redis nodes are running" >&2 - RESULT=1 - cleanup_and_exit -fi - -echo "Cluster Deployed....." -sleep 10 - -if [[ "${debug}" == "true" ]]; then - ./func_test.py --debug -else - sh -c ./func_test.py -fi -RESULT=$? -echo $RESULT - -# check a single stats port -curl -s localhost:22222/info | python -mjson.tool > /dev/null -if [[ $? -ne 0 ]]; then - echo "Stats are not working or not valid json" >&2 - RESULT=1 -fi - -DYNOMITE_NODES=`pgrep dynomite | wc -l` -REDIS_NODES=`pgrep redis-server | wc -l` - -if [[ $DYNOMITE_NODES -ne 5 ]]; then - echo "Not all dynomite nodes are running" >&2 - RESULT=1 - cleanup_and_exit -fi -if [[ $REDIS_NODES -ne 6 ]]; then - echo "Not all redis nodes are running" >&2 - RESULT=1 - cleanup_and_exit -fi - -cleanup_and_exit +exec test/cluster_generator.py