diff --git a/scripts/load_testing/README.md b/scripts/load_testing/README.md new file mode 100644 index 000000000..ad961f324 --- /dev/null +++ b/scripts/load_testing/README.md @@ -0,0 +1,30 @@ +## Examples + +For PMEM usage use `-m` parameter followed by a list of PMEM mount point paths. Without that parameter configs will be created for RAM usage. +Note that the first mount point provided will be bound to the numa node 0, next mount point to the numa node 1, etc. +Put a list of mount points in quotes when providing more than one path. + +Twemcache: + +``` +# generating configs without PMEM support +./generate.sh -s -p pelikan/_build/_bin/pelikan_twemcache -c -r rpc-perf/target/release/rpc-perf -t 127.0.0.1 + +# generating configs with PMEM support for two mount points +./generate.sh -s -p pelikan/_build/_bin/pelikan_twemcache -c -r rpc-perf/target/release/rpc-perf -t 127.0.0.1 -m "/mnt/pmem0 /mnt/pmem1" +``` + +Slimcache: + +``` +# generating configs without PMEM support +./generate.sh -s -p pelikan/_build/_bin/pelikan_slimcache -c -r rpc-perf/target/release/rpc-perf -t 127.0.0.1 + +# generating configs with PMEM support for two mount points +./generate.sh -s -p pelikan/_build/_bin/pelikan_slimcache -c -r rpc-perf/target/release/rpc-perf -t 127.0.0.1 -m "/mnt/pmem0 /mnt/pmem1" +``` + +To run benchmarks provide paths to generated config directories: +``` +./runtest.sh -c rpcperf_100_1024_4 -s pelikan_1024_4 -t 127.0.0.1 +``` diff --git a/scripts/load_testing/client_config.py b/scripts/load_testing/client_config.py index b094bb20f..2e4d8354d 100644 --- a/scripts/load_testing/client_config.py +++ b/scripts/load_testing/client_config.py @@ -60,7 +60,7 @@ def generate_runscript(binary, server_ip, instances): the_file.write(' --waterfall latency-waterfall-{server_port}.png'.format(server_port=server_port)) the_file.write(' > rpcperf_{server_port}.log'.format(server_port=server_port)) the_file.write(' 2>&1 &\n') - os.chmod(fname, 0777) + os.chmod(fname, 0o777) if __name__ == "__main__": diff --git a/scripts/load_testing/generate.sh b/scripts/load_testing/generate.sh index f68a17572..0e62a0a4d 100755 --- a/scripts/load_testing/generate.sh +++ b/scripts/load_testing/generate.sh @@ -14,15 +14,17 @@ server=false rpcperf="rpc-perf" pelikan="pelikan_twemcache" target="127.0.0.1" +pmem_paths=() show_help() { - echo "generate.sh [-c [-r path/to/rpcperf] [-t target/serverIP]] [-s [-p path/to/pelikan]]" + echo 'generate.sh [-c [-r path/to/rpcperf] [-t target/serverIP]] [-s [-p path/to/pelikan]] [-m "path/to/pmem0 path/to/pmem1 ..."]' + echo 'Note that the first pmem path is bound to the first numa node, the second path is bound to the next numa node. One or more paths can be provided.' } get_args() { - while getopts ":p:r:t:csh" opt; do + while getopts ":p:r:t:m:csh" opt; do case "$opt" in c) client=true ;; @@ -34,6 +36,8 @@ get_args() ;; t) target=$OPTARG ;; + m) pmem_paths=($OPTARG) + ;; h) show_help exit 0 @@ -57,7 +61,7 @@ gen_pelikan() do slab_mem=$((mem * 1024 * 1024 * 1024)) prefix=pelikan_${size}_${mem} - python server_config.py --prefix="$prefix" --binary="$pelikan" --instances="$instances" --slab_mem "$slab_mem" --vsize "$vsize" + python server_config.py --prefix="$prefix" --binary="$pelikan" --instances="$instances" --slab_mem "$slab_mem" --vsize "$vsize" --pmem_paths ${pmem_paths[@]} done done } @@ -87,4 +91,3 @@ fi if [ "$server" = true ]; then gen_pelikan fi - diff --git a/scripts/load_testing/server_config.py b/scripts/load_testing/server_config.py index c9b48660b..1f5ffd109 100644 --- a/scripts/load_testing/server_config.py +++ b/scripts/load_testing/server_config.py @@ -1,6 +1,8 @@ import argparse from math import ceil, floor, log import os +import subprocess +import sys INSTANCES = 3 PREFIX = 'test' @@ -13,8 +15,9 @@ THREAD_PER_SOCKET = 48 BIND_TO_CORES = False BIND_TO_NODES = True +ENGINE = "twemcache" -def generate_config(instances, vsize, slab_mem): +def generate_config(instances, vsize, slab_mem, pmem_paths, engine): # create top-level folders under prefix try: os.makedirs('config') @@ -25,14 +28,17 @@ def generate_config(instances, vsize, slab_mem): except: pass - nkey = int(ceil(1.0 * slab_mem / (vsize + KSIZE + PELIKAN_ITEM_OVERHEAD))) + item_size = vsize + KSIZE + PELIKAN_ITEM_OVERHEAD + nkey = int(ceil(1.0 * slab_mem / item_size)) hash_power = int(ceil(log(nkey, 2))) - # create twemcache config file(s) + # create twemcache|slimcache config file(s) for i in range(instances): admin_port = PELIKAN_ADMIN_PORT + i server_port = PELIKAN_SERVER_PORT + i - config_file = 'twemcache-{server_port}.config'.format(server_port=server_port) + config_file = '{engine}-{server_port}.config'.format(engine=engine, server_port=server_port) + + # String with common options for both twemcache and slimcache config_str = """\ daemonize: yes admin_port: {admin_port} @@ -45,11 +51,11 @@ def generate_config(instances, vsize, slab_mem): buf_sock_poolsize: 16384 debug_log_level: 5 -debug_log_file: log/twemcache-{server_port}.log +debug_log_file: log/{engine}-{server_port}.log debug_log_nbuf: 1048576 -klog_file: log/twemcache-{server_port}.cmd -klog_backup: log/twemcache-{server_port}.cmd.old +klog_file: log/{engine}-{server_port}.cmd +klog_backup: log/{engine}-{server_port}.cmd.old klog_sample: 100 klog_max: 1073741824 @@ -61,29 +67,57 @@ def generate_config(instances, vsize, slab_mem): request_poolsize: 16384 response_poolsize: 32768 +time_type: 2 +""".format(admin_port=admin_port, server_port=server_port, vsize=vsize, nkey=nkey, engine=engine) + + # String with options specific for either twemcache or slimcache + pmem_path_str = "" + datapool_param = "" + if engine == "slimcache": + datapool_param = "cuckoo_datapool" + engine_str = """\ + +cuckoo_item_size: {item_size} +cuckoo_nitem: {nkey} +cuckoo_datapool_prefault: yes +""".format(item_size=item_size, nkey=nkey) + elif engine == "twemcache": + datapool_param = "slab_datapool" + engine_str = """\ + slab_evict_opt: 1 slab_prealloc: yes slab_hash_power: {hash_power} slab_mem: {slab_mem} slab_size: 1048756 +slab_datapool_prefault: yes stats_intvl: 10000 stats_log_file: log/twemcache-{server_port}.stats +""".format(hash_power=hash_power, slab_mem=slab_mem, server_port=server_port) -time_type: 2 -""".format(admin_port=admin_port, server_port=server_port, vsize=vsize, nkey=nkey, hash_power=hash_power, slab_mem=slab_mem) + # String with option specific for PMEM usage + if len(pmem_paths) > 0: + pmem_path_str = """\ + +{datapool_param}: {pmem_path} +""".format(datapool_param=datapool_param, pmem_path=os.path.join(pmem_paths[i%len(pmem_paths)], 'pool_{}'.format(server_port))) + + # Put it all together + config_str = config_str + engine_str + pmem_path_str with open(os.path.join('config', config_file),'w') as the_file: the_file.write(config_str) -def generate_runscript(binary, instances): +def generate_runscript(binary, instances, pmem_paths_count, engine): # create bring-up.sh fname = 'bring-up.sh' + numa_node_count = pmem_paths_count if pmem_paths_count > 0 else 2 with open(fname, 'w') as the_file: for i in range(instances): - config_file = os.path.join('config', 'twemcache-{server_port}.config'.format(server_port=PELIKAN_SERVER_PORT+i)) + config_file = os.path.join('config', '{engine}-{server_port}.config'.format(engine=engine, server_port=PELIKAN_SERVER_PORT+i)) if BIND_TO_NODES: the_file.write('sudo numactl --cpunodebind={numa_node} --preferred={numa_node} '.format( - numa_node=i%2)) + numa_node=i%numa_node_count)) elif BIND_TO_CORES: the_file.write('sudo numactl --physcpubind={physical_thread},{logical_thread} '.format( physical_thread=i, @@ -91,10 +125,11 @@ def generate_runscript(binary, instances): the_file.write('{binary_file} {config_file}\n'.format( binary_file=binary, config_file=config_file)) - os.chmod(fname, 0777) + os.chmod(fname, 0o777) # create warm-up.sh fname = 'warm-up.sh' + prefill_opt = "prefilling cuckoo" if engine == "slimcache" else "prefilling slab" with open(fname, 'w') as the_file: the_file.write(""" ./bring-up.sh @@ -102,23 +137,24 @@ def generate_runscript(binary, instances): nready=0 while [ $nready -lt {instances} ] do - nready=$(grep -l "prefilling slab" log/twemcache-*.log | wc -l) + nready=$(grep -l "{prefill_opt}" log/{engine}-*.log | wc -l) echo "$(date): $nready out of {instances} servers are warmed up" sleep 10 done -""".format(instances=instances)) - os.chmod(fname, 0777) +""".format(instances=instances, prefill_opt=prefill_opt, engine=engine)) + os.chmod(fname, 0o777) if __name__ == "__main__": parser = argparse.ArgumentParser(description=""" Generate all the server-side scripts/configs needed for a test run. """) - parser.add_argument('--binary', dest='binary', type=str, help='location of pelikan_twemcache binary', required=True) + parser.add_argument('--binary', dest='binary', type=str, help='location of pelikan_twemcache|pelikan_slimcache binary', required=True) parser.add_argument('--prefix', dest='prefix', type=str, default=PREFIX, help='folder that contains all the other files to be generated') parser.add_argument('--instances', dest='instances', type=int, default=INSTANCES, help='number of instances') parser.add_argument('--vsize', dest='vsize', type=int, default=VSIZE, help='value size') parser.add_argument('--slab_mem', dest='slab_mem', type=int, default=PELIKAN_SLAB_MEM, help='total capacity of slab memory, in bytes') + parser.add_argument('--pmem_paths', dest='pmem_paths', nargs='*', help='list of pmem mount points') args = parser.parse_args() @@ -126,5 +162,14 @@ def generate_runscript(binary, instances): os.makedirs(args.prefix) os.chdir(args.prefix) - generate_config(args.instances, args.vsize, args.slab_mem) - generate_runscript(args.binary, args.instances) + binary_help_out = subprocess.check_output([args.binary, '--help']) + if binary_help_out.find("twemcache") != -1: + engine = "twemcache" + elif binary_help_out.find("slimcache") != -1: + engine = "slimcache" + else: + print('Provided binary is not twemcache|slimcache. Only these engines are valid. Exiting...') + sys.exit() + + generate_config(args.instances, args.vsize, args.slab_mem, args.pmem_paths, engine) + generate_runscript(args.binary, args.instances, len(args.pmem_paths), engine)