adding script to generate (stacked) server instances on a single host (…

…#206)

* adding script to generate (stacked) server instances on a single host

* fix missing prefix for bring-up.sh

* fix some bugs wrt file/dir creation

* adding client script

* add config option for key/val size (which adjusts hash_power value)

* adding bash script to generate server config groups

* making config file path relative/local to bring-up.sh

* using memcache style timestamp to work with rpcperf properly (which assumes that); improve key # calculation accuracy

* update server-side config generator with prefill options and minor fixes

* update client_config.py to match (server) prefilling pattern

* update client config parameters per Brian's suggestion

* improve client config; add comment; add waterfall chart output; add log output

* flattern log dir config for stacked server jobs

* add script to launch a single test

* make changes needed to launch runtest.sh from the client host

* uncomment client_run

* make binary & server_ip required parameters

* fix typo

* make rpc-perf binary location configurable too

* adapt client config for new rpc-perf which has different config syntax

* further parameterization

* misnomer corrected

* good day for typo

* more one word change

* restoring waterfall now that rpc-perf (prerelease) has the feature added back

* add stats log related config to server

* increase timing wheel size to allow 10 second timeout

scripts/load_testing/client_config.py

@@ -0,0 +1,87 @@
+import argparse
+from math import ceil
+import os
+
+
+INSTANCES = 3
+PREFIX = 'loadgen'
+RPCPERF_THREADS = 1
+RPCPERF_CONNS = 100
+RPCPERF_RATE = 10000
+RPCPERF_GET_WEIGHT = 9
+RPCPERF_SET_WEIGHT = 1
+PELIKAN_SLAB_MEM = 4294967296
+PELIKAN_ITEM_OVERHEAD = 48
+KSIZE = 32
+VSIZE = 32
+PELIKAN_SERVER_PORT = 12300
+
+
+def generate_config(rate, connections, vsize, slab_mem, threads):
+# create rpcperf.toml
+  nkey = int(ceil(1.0 * slab_mem / (vsize + KSIZE + PELIKAN_ITEM_OVERHEAD)))
+  conn_per_thread = connections / threads
+
+  config_str = '''
+[general]
+clients = {threads}
+tcp-nodelay = true
+poolsize = {connections} # this specifies number of connection per thread
+# runtime ~= windows x duration
+windows = 2
+interval = 60
+request_ratelimit = {rate}
+
+[[keyspace]]
+length = {ksize}
+count = {nkey}
+weight = 1
+commands = [
+    {{action = "get", weight = {get_weight}}},
+    {{action = "set", weight = {set_weight}}},
+]
+values = [
+    {{length = {vsize}, weight = 1}},
+]'''.format(threads=threads, connections=conn_per_thread, nkey=nkey, rate=rate,
+    ksize=KSIZE, vsize=vsize, get_weight=RPCPERF_GET_WEIGHT, set_weight=RPCPERF_SET_WEIGHT)
+
+  with open('rpcperf.toml', 'w') as the_file:
+    the_file.write(config_str)
+
+
+def generate_runscript(binary, server_ip, instances):
+  # create test.sh
+  fname = 'test.sh'
+  with open(fname, 'w') as the_file:
+    for i in range(instances):
+      server_port = PELIKAN_SERVER_PORT + i
+      the_file.write('{binary_file} --config {config_file}'.format(binary_file=binary, config_file='rpcperf.toml'))
+      the_file.write(' --endpoint {server_ip}:{server_port}'.format(server_ip=server_ip, server_port=server_port))
+      the_file.write(' --waterfall latency-waterfall-{server_port}.png'.format(server_port=server_port))
+      the_file.write(' > rpcperf_{server_port}.log'.format(server_port=server_port))
+      the_file.write(' 2>&1 &\n')
+  os.chmod(fname, 0777)
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="""
+    Generate all the client-side scripts/configs needed for a test run.
+    """)
+  parser.add_argument('--binary', dest='binary', type=str, help='location of rpc-perf binary', required=True)
+  parser.add_argument('--prefix', dest='prefix', type=str, default=PREFIX, help='folder that contains all the other files to be generated')
+  parser.add_argument('--instances', dest='instances', type=int, default=INSTANCES, help='number of instances')
+  parser.add_argument('--server_ip', dest='server_ip', type=str, help='server ip', required=True)
+  parser.add_argument('--rate', dest='rate', type=int, default=RPCPERF_RATE, help='request rate per instance')
+  parser.add_argument('--connections', dest='connections', type=int, default=RPCPERF_CONNS, help='number of connections per instance')
+  parser.add_argument('--vsize', dest='vsize', type=int, default=VSIZE, help='value size')
+  parser.add_argument('--slab_mem', dest='slab_mem', type=int, default=PELIKAN_SLAB_MEM, help='slab memory')
+  parser.add_argument('--threads', dest='threads', type=int, default=RPCPERF_THREADS, help='number of worker threads per rpc-perf')
+
+  args = parser.parse_args()
+
+  if not os.path.exists(args.prefix):
+    os.makedirs(args.prefix)
+  os.chdir(args.prefix)
+
+  generate_config(args.rate, args.connections, args.vsize, args.slab_mem, args.threads)
+  generate_runscript(args.binary, args.server_ip, args.instances)

scripts/load_testing/generate.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+conn_configs=(100 1000 10000)
+mem_configs=(4 8 16 32)
+size_configs=(64 128 256 512 1024 2048)
+instances=30
+ksize=32
+rate=100000
+threads=2
+
+# Initialize our own variables:
+client=false
+server=false
+rpcperf="rpc-perf"
+pelikan="pelikan_twemcache"
+target="127.0.0.1"
+
+show_help()
+{
+    echo "generate.sh [-c [-r path/to/rpcperf] [-t target/serverIP]] [-s [-p path/to/pelikan]]"
+}
+
+get_args()
+{
+    while getopts ":p:r:t:csh" opt; do
+        case "$opt" in
+        c)  client=true
+            ;;
+        s)  server=true
+            ;;
+        p)  pelikan=$OPTARG
+            ;;
+        r)  rpcperf=$OPTARG
+            ;;
+        t)  target=$OPTARG
+            ;;
+        h)
+            show_help
+            exit 0
+            ;;
+        \?)
+            echo "unrecognized option $opt"
+            show_help
+            exit 1
+            ;;
+        esac
+    done
+}
+
+# pelikan configs
+gen_pelikan()
+{
+    for size in "${size_configs[@]}"
+    do
+        vsize=$((size - ksize))
+        for mem in "${mem_configs[@]}"
+        do
+            slab_mem=$((mem * 1024 * 1024 * 1024))
+            prefix=pelikan_${size}_${mem}
+            python server_config.py --prefix="$prefix" --binary="$pelikan" --instances="$instances" --slab_mem "$slab_mem" --vsize "$vsize"
+        done
+    done
+}
+
+# rpc-perf configs
+gen_rpcperf()
+{
+    for conn in "${conn_configs[@]}"
+    do
+        for size in "${size_configs[@]}"
+        do
+            vsize=$((size - ksize))
+            for mem in "${mem_configs[@]}"
+            do
+                slab_mem=$((mem * 1024 * 1024 * 1024))
+                prefix=rpcperf_${conn}_${size}_${mem}
+                python client_config.py --prefix="$prefix" --binary="$rpcperf" --server_ip="$target" --instances="$instances" --rate="$rate" --connections="$conn" --vsize "$vsize" --slab_mem="$slab_mem" --threads="$threads"
+            done
+        done
+    done
+}
+
+get_args "${@}"
+if [ "$client" = true ]; then
+    gen_rpcperf
+fi
+if [ "$server" = true ]; then
+    gen_pelikan
+fi
+

scripts/load_testing/runtest.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# NOTE(!!): the script only works when all config folders are freshly created,
+# i.e. no data from previous runs
+
+# Initialize our own variables:
+client_config=""
+server_config=""
+target=""
+
+show_help()
+{
+    echo "runtest.sh -c <client_config_path> -s <server_config_path> -t <target: host where servers run>"
+}
+
+get_args()
+{
+    while getopts ":c:s:t:h" opt; do
+        case "$opt" in
+        c)  client_config=$OPTARG
+            ;;
+        s)  server_config=$OPTARG
+            ;;
+        t)  target=$OPTARG
+            ;;
+        h)
+            show_help
+            exit 0
+            ;;
+        \?)
+            echo "unrecognized option $opt"
+            show_help
+            exit 1
+            ;;
+        esac
+    done
+}
+
+server_launch()
+{
+    ssh -C "$target" "cd $server_config && ./warm-up.sh"
+}
+
+
+client_run()
+{
+    cd "$client_config" || exit 1
+
+    ./test.sh
+
+    local nrunning=1
+    while [ $nrunning -gt 0 ]
+    do
+        nrunning=$(pgrep -c rpc-perf)
+        echo "$(date): $nrunning clients are still running"
+        sleep 10
+    done
+
+    cd - > /dev/null || exit 1
+}
+
+wrap_up()
+{
+    ssh -C "$target" "pkill -f pelikan_twemcache"
+}
+
+get_args "${@}"
+server_launch
+client_run
+wrap_up

scripts/load_testing/server_config.py

@@ -0,0 +1,130 @@
+import argparse
+from math import ceil, floor, log
+import os
+
+INSTANCES = 3
+PREFIX = 'test'
+PELIKAN_ADMIN_PORT = 9900
+PELIKAN_SERVER_PORT = 12300
+PELIKAN_SLAB_MEM = 4294967296
+PELIKAN_ITEM_OVERHEAD = 48
+KSIZE = 32
+VSIZE = 32
+THREAD_PER_SOCKET = 48
+BIND_TO_CORES = False
+BIND_TO_NODES = True
+
+def generate_config(instances, vsize, slab_mem):
+  # create top-level folders under prefix
+  try:
+    os.makedirs('config')
+  except:
+    pass
+  try:
+    os.makedirs('log')
+  except:
+    pass
+
+  nkey = int(ceil(1.0 * slab_mem / (vsize + KSIZE + PELIKAN_ITEM_OVERHEAD)))
+  hash_power = int(ceil(log(nkey, 2)))
+
+  # create twemcache config file(s)
+  for i in range(instances):
+    admin_port = PELIKAN_ADMIN_PORT + i
+    server_port = PELIKAN_SERVER_PORT + i
+    config_file = 'twemcache-{server_port}.config'.format(server_port=server_port)
+    config_str = """\
+daemonize: yes
+admin_port: {admin_port}
+server_port: {server_port}
+
+admin_tw_cap: 2000
+
+buf_init_size: 4096
+
+buf_sock_poolsize: 16384
+
+debug_log_level: 5
+debug_log_file: log/twemcache-{server_port}.log
+debug_log_nbuf: 1048576
+
+klog_file: log/twemcache-{server_port}.cmd
+klog_backup: log/twemcache-{server_port}.cmd.old
+klog_sample: 100
+klog_max: 1073741824
+
+prefill: yes
+prefill_ksize: 32
+prefill_vsize: {vsize}
+prefill_nkey: {nkey}
+
+request_poolsize: 16384
+response_poolsize: 32768
+
+slab_evict_opt: 1
+slab_prealloc: yes
+slab_hash_power: {hash_power}
+slab_mem: {slab_mem}
+slab_size: 1048756
+
+stats_intvl: 10000
+stats_log_file: log/twemcache-{server_port}.stats
+
+time_type: 2
+""".format(admin_port=admin_port, server_port=server_port, vsize=vsize, nkey=nkey, hash_power=hash_power, slab_mem=slab_mem)
+    with open(os.path.join('config', config_file),'w') as the_file:
+      the_file.write(config_str)
+
+def generate_runscript(binary, instances):
+  # create bring-up.sh
+  fname = 'bring-up.sh'
+  with open(fname, 'w') as the_file:
+    for i in range(instances):
+      config_file = os.path.join('config', 'twemcache-{server_port}.config'.format(server_port=PELIKAN_SERVER_PORT+i))
+      if BIND_TO_NODES:
+        the_file.write('sudo numactl --cpunodebind={numa_node} --preferred={numa_node} '.format(
+            numa_node=i%2))
+      elif BIND_TO_CORES:
+        the_file.write('sudo numactl --physcpubind={physical_thread},{logical_thread} '.format(
+            physical_thread=i,
+            logical_thread=i+THREAD_PER_SOCKET))
+      the_file.write('{binary_file} {config_file}\n'.format(
+          binary_file=binary,
+          config_file=config_file))
+  os.chmod(fname, 0777)
+
+  # create warm-up.sh
+  fname = 'warm-up.sh'
+  with open(fname, 'w') as the_file:
+    the_file.write("""
+./bring-up.sh
+
+nready=0
+while [ $nready -lt {instances} ]
+do
+    nready=$(grep -l "prefilling slab" log/twemcache-*.log | wc -l)
+    echo "$(date): $nready out of {instances} servers are warmed up"
+    sleep 10
+done
+""".format(instances=instances))
+  os.chmod(fname, 0777)
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="""
+    Generate all the server-side scripts/configs needed for a test run.
+    """)
+  parser.add_argument('--binary', dest='binary', type=str, help='location of pelikan_twemcache binary', required=True)
+  parser.add_argument('--prefix', dest='prefix', type=str, default=PREFIX, help='folder that contains all the other files to be generated')
+  parser.add_argument('--instances', dest='instances', type=int, default=INSTANCES, help='number of instances')
+  parser.add_argument('--vsize', dest='vsize', type=int, default=VSIZE, help='value size')
+  parser.add_argument('--slab_mem', dest='slab_mem', type=int, default=PELIKAN_SLAB_MEM, help='total capacity of slab memory, in bytes')
+
+  args = parser.parse_args()
+
+  if not os.path.exists(args.prefix):
+    os.makedirs(args.prefix)
+  os.chdir(args.prefix)
+
+  generate_config(args.instances, args.vsize, args.slab_mem)
+  generate_runscript(args.binary, args.instances)