From 989822e0dbc2045ff2e5550a6e3c2b8996d5ba8c Mon Sep 17 00:00:00 2001 From: Yao Yue Date: Thu, 11 Jul 2019 10:08:59 -0700 Subject: [PATCH] adding script to generate (stacked) server instances on a single host (#206) * adding script to generate (stacked) server instances on a single host * fix missing prefix for bring-up.sh * fix some bugs wrt file/dir creation * adding client script * add config option for key/val size (which adjusts hash_power value) * adding bash script to generate server config groups * making config file path relative/local to bring-up.sh * using memcache style timestamp to work with rpcperf properly (which assumes that); improve key # calculation accuracy * update server-side config generator with prefill options and minor fixes * update client_config.py to match (server) prefilling pattern * update client config parameters per Brian's suggestion * improve client config; add comment; add waterfall chart output; add log output * flattern log dir config for stacked server jobs * add script to launch a single test * make changes needed to launch runtest.sh from the client host * uncomment client_run * make binary & server_ip required parameters * fix typo * make rpc-perf binary location configurable too * adapt client config for new rpc-perf which has different config syntax * further parameterization * misnomer corrected * good day for typo * more one word change * restoring waterfall now that rpc-perf (prerelease) has the feature added back * add stats log related config to server * increase timing wheel size to allow 10 second timeout --- scripts/load_testing/client_config.py | 87 +++++++++++++++++ scripts/load_testing/generate.sh | 90 ++++++++++++++++++ scripts/load_testing/runtest.sh | 70 ++++++++++++++ scripts/load_testing/server_config.py | 130 ++++++++++++++++++++++++++ 4 files changed, 377 insertions(+) create mode 100644 scripts/load_testing/client_config.py create mode 100755 scripts/load_testing/generate.sh create mode 100755 scripts/load_testing/runtest.sh create mode 100644 scripts/load_testing/server_config.py diff --git a/scripts/load_testing/client_config.py b/scripts/load_testing/client_config.py new file mode 100644 index 000000000..b094bb20f --- /dev/null +++ b/scripts/load_testing/client_config.py @@ -0,0 +1,87 @@ +import argparse +from math import ceil +import os + + +INSTANCES = 3 +PREFIX = 'loadgen' +RPCPERF_THREADS = 1 +RPCPERF_CONNS = 100 +RPCPERF_RATE = 10000 +RPCPERF_GET_WEIGHT = 9 +RPCPERF_SET_WEIGHT = 1 +PELIKAN_SLAB_MEM = 4294967296 +PELIKAN_ITEM_OVERHEAD = 48 +KSIZE = 32 +VSIZE = 32 +PELIKAN_SERVER_PORT = 12300 + + +def generate_config(rate, connections, vsize, slab_mem, threads): +# create rpcperf.toml + nkey = int(ceil(1.0 * slab_mem / (vsize + KSIZE + PELIKAN_ITEM_OVERHEAD))) + conn_per_thread = connections / threads + + config_str = ''' +[general] +clients = {threads} +tcp-nodelay = true +poolsize = {connections} # this specifies number of connection per thread +# runtime ~= windows x duration +windows = 2 +interval = 60 +request_ratelimit = {rate} + +[[keyspace]] +length = {ksize} +count = {nkey} +weight = 1 +commands = [ + {{action = "get", weight = {get_weight}}}, + {{action = "set", weight = {set_weight}}}, +] +values = [ + {{length = {vsize}, weight = 1}}, +]'''.format(threads=threads, connections=conn_per_thread, nkey=nkey, rate=rate, + ksize=KSIZE, vsize=vsize, get_weight=RPCPERF_GET_WEIGHT, set_weight=RPCPERF_SET_WEIGHT) + + with open('rpcperf.toml', 'w') as the_file: + the_file.write(config_str) + + +def generate_runscript(binary, server_ip, instances): + # create test.sh + fname = 'test.sh' + with open(fname, 'w') as the_file: + for i in range(instances): + server_port = PELIKAN_SERVER_PORT + i + the_file.write('{binary_file} --config {config_file}'.format(binary_file=binary, config_file='rpcperf.toml')) + the_file.write(' --endpoint {server_ip}:{server_port}'.format(server_ip=server_ip, server_port=server_port)) + the_file.write(' --waterfall latency-waterfall-{server_port}.png'.format(server_port=server_port)) + the_file.write(' > rpcperf_{server_port}.log'.format(server_port=server_port)) + the_file.write(' 2>&1 &\n') + os.chmod(fname, 0777) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=""" + Generate all the client-side scripts/configs needed for a test run. + """) + parser.add_argument('--binary', dest='binary', type=str, help='location of rpc-perf binary', required=True) + parser.add_argument('--prefix', dest='prefix', type=str, default=PREFIX, help='folder that contains all the other files to be generated') + parser.add_argument('--instances', dest='instances', type=int, default=INSTANCES, help='number of instances') + parser.add_argument('--server_ip', dest='server_ip', type=str, help='server ip', required=True) + parser.add_argument('--rate', dest='rate', type=int, default=RPCPERF_RATE, help='request rate per instance') + parser.add_argument('--connections', dest='connections', type=int, default=RPCPERF_CONNS, help='number of connections per instance') + parser.add_argument('--vsize', dest='vsize', type=int, default=VSIZE, help='value size') + parser.add_argument('--slab_mem', dest='slab_mem', type=int, default=PELIKAN_SLAB_MEM, help='slab memory') + parser.add_argument('--threads', dest='threads', type=int, default=RPCPERF_THREADS, help='number of worker threads per rpc-perf') + + args = parser.parse_args() + + if not os.path.exists(args.prefix): + os.makedirs(args.prefix) + os.chdir(args.prefix) + + generate_config(args.rate, args.connections, args.vsize, args.slab_mem, args.threads) + generate_runscript(args.binary, args.server_ip, args.instances) diff --git a/scripts/load_testing/generate.sh b/scripts/load_testing/generate.sh new file mode 100755 index 000000000..f68a17572 --- /dev/null +++ b/scripts/load_testing/generate.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +conn_configs=(100 1000 10000) +mem_configs=(4 8 16 32) +size_configs=(64 128 256 512 1024 2048) +instances=30 +ksize=32 +rate=100000 +threads=2 + +# Initialize our own variables: +client=false +server=false +rpcperf="rpc-perf" +pelikan="pelikan_twemcache" +target="127.0.0.1" + +show_help() +{ + echo "generate.sh [-c [-r path/to/rpcperf] [-t target/serverIP]] [-s [-p path/to/pelikan]]" +} + +get_args() +{ + while getopts ":p:r:t:csh" opt; do + case "$opt" in + c) client=true + ;; + s) server=true + ;; + p) pelikan=$OPTARG + ;; + r) rpcperf=$OPTARG + ;; + t) target=$OPTARG + ;; + h) + show_help + exit 0 + ;; + \?) + echo "unrecognized option $opt" + show_help + exit 1 + ;; + esac + done +} + +# pelikan configs +gen_pelikan() +{ + for size in "${size_configs[@]}" + do + vsize=$((size - ksize)) + for mem in "${mem_configs[@]}" + do + slab_mem=$((mem * 1024 * 1024 * 1024)) + prefix=pelikan_${size}_${mem} + python server_config.py --prefix="$prefix" --binary="$pelikan" --instances="$instances" --slab_mem "$slab_mem" --vsize "$vsize" + done + done +} + +# rpc-perf configs +gen_rpcperf() +{ + for conn in "${conn_configs[@]}" + do + for size in "${size_configs[@]}" + do + vsize=$((size - ksize)) + for mem in "${mem_configs[@]}" + do + slab_mem=$((mem * 1024 * 1024 * 1024)) + prefix=rpcperf_${conn}_${size}_${mem} + python client_config.py --prefix="$prefix" --binary="$rpcperf" --server_ip="$target" --instances="$instances" --rate="$rate" --connections="$conn" --vsize "$vsize" --slab_mem="$slab_mem" --threads="$threads" + done + done + done +} + +get_args "${@}" +if [ "$client" = true ]; then + gen_rpcperf +fi +if [ "$server" = true ]; then + gen_pelikan +fi + diff --git a/scripts/load_testing/runtest.sh b/scripts/load_testing/runtest.sh new file mode 100755 index 000000000..d5d63ad63 --- /dev/null +++ b/scripts/load_testing/runtest.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# NOTE(!!): the script only works when all config folders are freshly created, +# i.e. no data from previous runs + +# Initialize our own variables: +client_config="" +server_config="" +target="" + +show_help() +{ + echo "runtest.sh -c -s -t " +} + +get_args() +{ + while getopts ":c:s:t:h" opt; do + case "$opt" in + c) client_config=$OPTARG + ;; + s) server_config=$OPTARG + ;; + t) target=$OPTARG + ;; + h) + show_help + exit 0 + ;; + \?) + echo "unrecognized option $opt" + show_help + exit 1 + ;; + esac + done +} + +server_launch() +{ + ssh -C "$target" "cd $server_config && ./warm-up.sh" +} + + +client_run() +{ + cd "$client_config" || exit 1 + + ./test.sh + + local nrunning=1 + while [ $nrunning -gt 0 ] + do + nrunning=$(pgrep -c rpc-perf) + echo "$(date): $nrunning clients are still running" + sleep 10 + done + + cd - > /dev/null || exit 1 +} + +wrap_up() +{ + ssh -C "$target" "pkill -f pelikan_twemcache" +} + +get_args "${@}" +server_launch +client_run +wrap_up diff --git a/scripts/load_testing/server_config.py b/scripts/load_testing/server_config.py new file mode 100644 index 000000000..c9b48660b --- /dev/null +++ b/scripts/load_testing/server_config.py @@ -0,0 +1,130 @@ +import argparse +from math import ceil, floor, log +import os + +INSTANCES = 3 +PREFIX = 'test' +PELIKAN_ADMIN_PORT = 9900 +PELIKAN_SERVER_PORT = 12300 +PELIKAN_SLAB_MEM = 4294967296 +PELIKAN_ITEM_OVERHEAD = 48 +KSIZE = 32 +VSIZE = 32 +THREAD_PER_SOCKET = 48 +BIND_TO_CORES = False +BIND_TO_NODES = True + +def generate_config(instances, vsize, slab_mem): + # create top-level folders under prefix + try: + os.makedirs('config') + except: + pass + try: + os.makedirs('log') + except: + pass + + nkey = int(ceil(1.0 * slab_mem / (vsize + KSIZE + PELIKAN_ITEM_OVERHEAD))) + hash_power = int(ceil(log(nkey, 2))) + + # create twemcache config file(s) + for i in range(instances): + admin_port = PELIKAN_ADMIN_PORT + i + server_port = PELIKAN_SERVER_PORT + i + config_file = 'twemcache-{server_port}.config'.format(server_port=server_port) + config_str = """\ +daemonize: yes +admin_port: {admin_port} +server_port: {server_port} + +admin_tw_cap: 2000 + +buf_init_size: 4096 + +buf_sock_poolsize: 16384 + +debug_log_level: 5 +debug_log_file: log/twemcache-{server_port}.log +debug_log_nbuf: 1048576 + +klog_file: log/twemcache-{server_port}.cmd +klog_backup: log/twemcache-{server_port}.cmd.old +klog_sample: 100 +klog_max: 1073741824 + +prefill: yes +prefill_ksize: 32 +prefill_vsize: {vsize} +prefill_nkey: {nkey} + +request_poolsize: 16384 +response_poolsize: 32768 + +slab_evict_opt: 1 +slab_prealloc: yes +slab_hash_power: {hash_power} +slab_mem: {slab_mem} +slab_size: 1048756 + +stats_intvl: 10000 +stats_log_file: log/twemcache-{server_port}.stats + +time_type: 2 +""".format(admin_port=admin_port, server_port=server_port, vsize=vsize, nkey=nkey, hash_power=hash_power, slab_mem=slab_mem) + with open(os.path.join('config', config_file),'w') as the_file: + the_file.write(config_str) + +def generate_runscript(binary, instances): + # create bring-up.sh + fname = 'bring-up.sh' + with open(fname, 'w') as the_file: + for i in range(instances): + config_file = os.path.join('config', 'twemcache-{server_port}.config'.format(server_port=PELIKAN_SERVER_PORT+i)) + if BIND_TO_NODES: + the_file.write('sudo numactl --cpunodebind={numa_node} --preferred={numa_node} '.format( + numa_node=i%2)) + elif BIND_TO_CORES: + the_file.write('sudo numactl --physcpubind={physical_thread},{logical_thread} '.format( + physical_thread=i, + logical_thread=i+THREAD_PER_SOCKET)) + the_file.write('{binary_file} {config_file}\n'.format( + binary_file=binary, + config_file=config_file)) + os.chmod(fname, 0777) + + # create warm-up.sh + fname = 'warm-up.sh' + with open(fname, 'w') as the_file: + the_file.write(""" +./bring-up.sh + +nready=0 +while [ $nready -lt {instances} ] +do + nready=$(grep -l "prefilling slab" log/twemcache-*.log | wc -l) + echo "$(date): $nready out of {instances} servers are warmed up" + sleep 10 +done +""".format(instances=instances)) + os.chmod(fname, 0777) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=""" + Generate all the server-side scripts/configs needed for a test run. + """) + parser.add_argument('--binary', dest='binary', type=str, help='location of pelikan_twemcache binary', required=True) + parser.add_argument('--prefix', dest='prefix', type=str, default=PREFIX, help='folder that contains all the other files to be generated') + parser.add_argument('--instances', dest='instances', type=int, default=INSTANCES, help='number of instances') + parser.add_argument('--vsize', dest='vsize', type=int, default=VSIZE, help='value size') + parser.add_argument('--slab_mem', dest='slab_mem', type=int, default=PELIKAN_SLAB_MEM, help='total capacity of slab memory, in bytes') + + args = parser.parse_args() + + if not os.path.exists(args.prefix): + os.makedirs(args.prefix) + os.chdir(args.prefix) + + generate_config(args.instances, args.vsize, args.slab_mem) + generate_runscript(args.binary, args.instances)