adding script to generate (stacked) server instances on a single host

scripts/load_testing/client_config.py

@@ -0,0 +1,101 @@
+import argparse
+from math import ceil
+import os
+
+
+INSTANCES = 3
+PREFIX = 'loadgen'
+RPCPERF_THREADS = 1
+RPCPERF_CONNS = 100
+RPCPERF_RATE = 10000
+RPCPERF_GET_RATIO = 0.9
+RPCPERF_SET_RATIO = 1 - RPCPERF_GET_RATIO
+PELIKAN_SLAB_MEM = 4294967296
+PELIKAN_ITEM_OVERHEAD = 48
+KSIZE = 32
+VSIZE = 32
+PELIKAN_SERVER_PORT = 12300
+PELIKAN_SERVER_IP = '10.25.2.44'
+RPCPERF_BINARY = '/root/Twitter/rpc-perf/target/release/rpc-perf'
+
+
+def generate_config(rate, connections, vsize, slab_mem, threads):
+# create rpcperf.toml
+  rate_get = int(rate * RPCPERF_GET_RATIO)
+  rate_set = int(rate * RPCPERF_SET_RATIO)
+  nkey = int(ceil(1.0 * slab_mem / (vsize + KSIZE + PELIKAN_ITEM_OVERHEAD)))
+  conn_per_thread = connections / threads
+
+  config_str = """\
+[general]
+threads = {threads}
+tcp-nodelay = true
+connections = {connections} # this specifies number of connection per thread
+# runtime ~= windows x duration
+windows = 1
+duration = 600
+request-timeout = 200
+connect-timeout = 250
+
+[[workload]]
+name = "get"
+method = "get"
+rate = {rate_get}
+  [[workload.parameter]]
+  style = "random"
+  size = 32
+  num = {nkey}
+  regenerate = true
+
+[[workload]]
+name = "set"
+method = "set"
+rate = {rate_set}
+  [[workload.parameter]] # key generation parameters
+  style = "random"
+  size = 32
+  num = {nkey}
+  regenerate = true
+  [[workload.parameter]] # value generation parameters
+  style = "random"
+  size = {vsize}
+  regenerate = false
+""".format(threads=threads, connections=conn_per_thread, nkey=nkey, rate_get=rate_get, rate_set=rate_set, vsize=vsize)
+  with open('rpcperf.toml', 'w') as the_file:
+    the_file.write(config_str)
+
+
+def generate_runscript(instances):
+  # create test.sh
+  fname = 'test.sh'
+  with open(fname, 'w') as the_file:
+    for i in range(instances):
+      server_port = PELIKAN_SERVER_PORT + i
+      the_file.write('{binary_file} --config {config_file}'.format(binary_file=RPCPERF_BINARY, config_file='rpcperf.toml'))
+      the_file.write(' --server {server_ip}:{server_port}'.format(server_ip=PELIKAN_SERVER_IP, server_port=server_port))
+      the_file.write(' --waterfall latency-waterfall-{server_port}.png'.format(server_port=server_port))
+      the_file.write(' > rpcperf_{server_port}.log'.format(server_port=server_port))
+      the_file.write(' 2>&1 &\n')
+  os.chmod(fname, 0777)
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="""
+    Generate all the client-side scripts/configs needed for a test run.
+    """)
+  parser.add_argument('--prefix', dest='prefix', type=str, default=PREFIX, help='folder that contains all the other files to be generated')
+  parser.add_argument('--instances', dest='instances', type=int, default=INSTANCES, help='number of instances')
+  parser.add_argument('--rate', dest='rate', type=int, default=RPCPERF_RATE, help='request rate per instance')
+  parser.add_argument('--connections', dest='connections', type=int, default=RPCPERF_CONNS, help='number of connections per instance')
+  parser.add_argument('--vsize', dest='vsize', type=int, default=VSIZE, help='value size')
+  parser.add_argument('--slab_mem', dest='slab_mem', type=int, default=PELIKAN_SLAB_MEM, help='slab memory')
+  parser.add_argument('--threads', dest='threads', type=int, default=RPCPERF_THREADS, help='number of worker threads per rpc-perf')
+
+  args = parser.parse_args()
+
+  if not os.path.exists(args.prefix):
+    os.makedirs(args.prefix)
+  os.chdir(args.prefix)
+
+  generate_config(args.rate, args.connections, args.vsize, args.slab_mem, args.threads)
+  generate_runscript(args.instances)

scripts/load_testing/generate.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+conn_configs=(100 1000 10000)
+mem_configs=(4 8 16 32)
+size_configs=(64 128 256 512 1024 2048)
+instances=30
+ksize=32
+rate=100000
+threads=2
+
+# pelikan configs
+for size in "${size_configs[@]}"
+do
+    vsize=$((size - ksize))
+    for mem in "${mem_configs[@]}"
+    do
+        slab_mem=$((mem * 1024 * 1024 * 1024))
+        prefix=pelikan_${size}_${mem}
+        python server_config.py --prefix="$prefix" --instances="$instances" --slab_mem "$slab_mem" --vsize "$vsize"
+    done
+done
+
+# rpc-perf configs
+for conn in "${conn_configs[@]}"
+do
+    for size in "${size_configs[@]}"
+    do
+        vsize=$((size - ksize))
+        for mem in "${mem_configs[@]}"
+        do
+            slab_mem=$((mem * 1024 * 1024 * 1024))
+            prefix=rpcperf_${conn}_${size}_${mem}
+            python client_config.py --prefix="$prefix" --instances="$instances" --rate="$rate" --connections="$conn" --vsize "$vsize" --slab_mem="$slab_mem" --threads="$threads"
+        done
+    done
+done

scripts/load_testing/runtest.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# NOTE(!!): the script only works when all config folders are freshly created,
+# i.e. no data from previous runs
+
+# Initialize our own variables:
+client_config=""
+server_config=""
+target=""
+
+show_help()
+{
+    echo "runtest.sh -c <client_config_path> -s <server_config_path> -t <target: host where servers run>"
+}
+
+get_args()
+{
+    while getopts ":c:s:t:h" opt; do
+        case "$opt" in
+        c)  client_config=$OPTARG
+            ;;
+        s)  server_config=$OPTARG
+            ;;
+        t)  target=$OPTARG
+            ;;
+        h)
+            show_help
+            exit 0
+            ;;
+        \?)
+            echo "unrecognized option $opt"
+            show_help
+            exit 1
+            ;;
+        esac
+    done
+}
+
+server_launch()
+{
+    ssh -C "$target" "cd $server_config && ./warm-up.sh"
+}
+
+
+client_run()
+{
+    cd "$client_config" || exit 1
+
+    ./test.sh
+
+    local nrunning=1
+    while [ $nrunning -gt 0 ]
+    do
+        nrunning=$(pgrep -c rpc-perf)
+        echo "$(date): $nrunning clients are still running"
+        sleep 10
+    done
+
+    cd - > /dev/null || exit 1
+}
+
+wrap_up()
+{
+    ssh -C "$target" "pkill -f pelikan_twemcache"
+}
+
+get_args "${@}"
+server_launch
+client_run
+wrap_up

scripts/load_testing/server_config.py

@@ -0,0 +1,126 @@
+import argparse
+from math import ceil, floor, log
+import os
+
+INSTANCES = 3
+PREFIX = 'test'
+PELIKAN_ADMIN_PORT = 9900
+PELIKAN_SERVER_PORT = 12300
+PELIKAN_SERVER_IP = '10.25.2.45'
+PELIKAN_SLAB_MEM = 4294967296
+PELIKAN_ITEM_OVERHEAD = 48
+PELIKAN_BINARY = '/root/Twitter/pelikan/_build/_bin/pelikan_twemcache'
+KSIZE = 32
+VSIZE = 32
+THREAD_PER_SOCKET = 48
+BIND_TO_CORES = False
+BIND_TO_NODES = True
+
+def generate_config(instances, vsize, slab_mem):
+  # create top-level folders under prefix
+  try:
+    os.makedirs('config')
+  except:
+    pass
+  try:
+    os.makedirs('log')
+  except:
+    pass
+
+  nkey = int(ceil(1.0 * slab_mem / (vsize + KSIZE + PELIKAN_ITEM_OVERHEAD)))
+  hash_power = int(ceil(log(nkey, 2)))
+
+  # create twemcache config file(s)
+  for i in range(instances):
+    admin_port = PELIKAN_ADMIN_PORT + i
+    server_port = PELIKAN_SERVER_PORT + i
+    config_file = 'twemcache-{server_port}.config'.format(server_port=server_port)
+    config_str = """\
+daemonize: yes
+admin_port: {admin_port}
+server_port: {server_port}
+
+buf_init_size: 4096
+
+buf_sock_poolsize: 16384
+
+debug_log_level: 5
+debug_log_file: log/twemcache-{server_port}.log
+debug_log_nbuf: 1048576
+
+klog_file: log/twemcache-{server_port}.cmd
+klog_backup: log/twemcache-{server_port}.cmd.old
+klog_sample: 100
+klog_max: 1073741824
+
+prefill: yes
+prefill_ksize: 32
+prefill_vsize: {vsize}
+prefill_nkey: {nkey}
+
+request_poolsize: 16384
+response_poolsize: 32768
+
+slab_evict_opt: 1
+slab_prealloc: yes
+slab_hash_power: {hash_power}
+slab_mem: {slab_mem}
+slab_size: 1048756
+
+time_type: 2
+""".format(admin_port=admin_port, server_port=server_port, vsize=vsize, nkey=nkey, hash_power=hash_power, slab_mem=slab_mem)
+    with open(os.path.join('config', config_file),'w') as the_file:
+      the_file.write(config_str)
+
+def generate_runscript(instances):
+  # create bring-up.sh
+  fname = 'bring-up.sh'
+  with open(fname, 'w') as the_file:
+    for i in range(instances):
+      config_file = os.path.join('config', 'twemcache-{server_port}.config'.format(server_port=PELIKAN_SERVER_PORT+i))
+      if BIND_TO_NODES:
+        the_file.write('sudo numactl --cpunodebind={numa_node} --preferred={numa_node} '.format(
+            numa_node=i%2))
+      elif BIND_TO_CORES:
+        the_file.write('sudo numactl --physcpubind={physical_thread},{logical_thread} '.format(
+            physical_thread=i,
+            logical_thread=i+THREAD_PER_SOCKET))
+      the_file.write('{binary_file} {config_file}\n'.format(
+          binary_file=PELIKAN_BINARY,
+          config_file=config_file))
+  os.chmod(fname, 0777)
+
+  # create warm-up.sh
+  fname = 'warm-up.sh'
+  with open(fname, 'w') as the_file:
+    the_file.write("""
+./bring-up.sh
+
+nready=0
+while [ $nready -lt {instances} ]
+do
+    nready=$(grep -l "prefilling slab" log/twemcache-*.log | wc -l)
+    echo "$(date): $nready out of {instances} servers are warmed up"
+    sleep 10
+done
+""".format(instances=instances))
+  os.chmod(fname, 0777)
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="""
+    Generate all the server-side scripts/configs needed for a test run.
+    """)
+  parser.add_argument('--prefix', dest='prefix', type=str, default=PREFIX, help='folder that contains all the other files to be generated')
+  parser.add_argument('--instances', dest='instances', type=int, default=INSTANCES, help='number of instances')
+  parser.add_argument('--vsize', dest='vsize', type=int, default=VSIZE, help='value size')
+  parser.add_argument('--slab_mem', dest='slab_mem', type=int, default=PELIKAN_SLAB_MEM, help='total capacity of slab memory, in bytes')
+
+  args = parser.parse_args()
+
+  if not os.path.exists(args.prefix):
+    os.makedirs(args.prefix)
+  os.chdir(args.prefix)
+
+  generate_config(args.instances, args.vsize, args.slab_mem)
+  generate_runscript(args.instances)