From 9ba9eadffa913774fb9e87e7edd577d5b9cb917d Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Thu, 29 Nov 2018 18:53:37 -0800
Subject: [PATCH 01/27] adding script to generate (stacked) server instances on
 a single host

---
 scripts/load_testing/server_config.py | 90 +++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)
 create mode 100644 scripts/load_testing/server_config.py

diff --git a/scripts/load_testing/server_config.py b/scripts/load_testing/server_config.py
new file mode 100644
index 000000000..d23d73766
--- /dev/null
+++ b/scripts/load_testing/server_config.py
@@ -0,0 +1,90 @@
+import os
+import argparse
+
+INSTANCES = 3
+PREFIX = 'test'
+PELIKAN_ADMIN_PORT = 9900
+PELIKAN_SERVER_PORT = 12300
+PELIKAN_SERVER_IP = '10.25.2.45'
+PELIKAN_SLAB_MEM = 4294967296
+PELIKAN_BINARY = '/root/Twitter/pelikan/_build/_bin/pelikan_twemcache'
+THREAD_PER_SOCKET = 48
+BIND_TO_CORES = False
+BIND_TO_NODES = True
+
+def generate_config(prefix, instances, slab_mem):
+  # create top-level folders under prefix
+  config_path = os.path.join(prefix, 'config')
+  os.makedirs(config_path)
+  log_path = os.path.join(prefix, 'log')
+  os.makedirs(log_path)
+
+  # create twemcache config file(s)
+  for i in range(instances):
+    admin_port = PELIKAN_ADMIN_PORT + i
+    server_port = PELIKAN_SERVER_PORT + i
+    config_file = 'pelikan-{server_port}.config'.format(server_port=server_port)
+    config_str = """\
+daemonize: yes
+admin_port: {admin_port}
+server_port: {server_port}
+
+buf_init_size: 4096
+
+buf_sock_poolsize: 16384
+
+debug_log_level: 5
+debug_log_file: log/{server_port}/twemcache.log
+debug_log_nbuf: 1048576
+
+klog_file: log/{server_port}/twemcache.cmd
+klog_backup: log/{server_port}/twemcache.cmd.old
+klog_sample: 100
+klog_max: 1073741824
+
+request_poolsize: 16384
+response_poolsize: 32768
+
+slab_evict_opt: 1
+slab_prealloc: yes
+slab_hash_power: 26
+slab_mem: {slab_mem}
+slab_size: 1048756
+""".format(admin_port=admin_port, server_port=server_port, slab_mem=slab_mem)
+    try:
+      os.makedirs(os.path.join(log_path, str(server_port)))
+    except:
+      pass
+    with open(os.path.join(config_path, config_file),'w') as the_file:
+      the_file.write(config_str)
+
+def generate_runscript(prefix, instances):
+  config_path = os.path.join(prefix, 'config')
+  # create bring-up.sh
+  with open('bring-up.sh','w') as the_file:
+    for i in range(instances):
+      config_file = os.path.join(config_path, 'pelikan-{server_port}.config'.format(server_port=PELIKAN_SERVER_PORT+i))
+      if BIND_TO_NODES:
+        the_file.write('sudo numactl --cpunodebind={numa_node} --preferred={numa_node} '.format(
+            numa_node=i%2))
+      elif BIND_TO_CORES:
+        the_file.write('sudo numactl --physcpubind={physical_thread},{logical_thread} '.format(
+            physical_thread=i,
+            logical_thread=i+THREAD_PER_SOCKET))
+      the_file.write('{binary_file} {config_file}\n'.format(
+          binary_file=PELIKAN_BINARY,
+          config_file=config_file))
+  os.chmod('bring-up.sh', 0777)
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="""
+    Generate all the server-side scripts/configs needed for a test run.
+    """)
+  parser.add_argument('--prefix', dest='prefix', type=str, default=PREFIX, help='folder that contains all the other files to be generated')
+  parser.add_argument('--instances', dest='instances', type=int, default=INSTANCES, help='number of instances')
+  parser.add_argument('--slab_mem', dest='slab_mem', type=int, default=PELIKAN_SLAB_MEM, help='total capacity of slab memory, in bytes')
+
+  args = parser.parse_args()
+
+  generate_config(args.prefix, args.instances, args.slab_mem)
+  generate_runscript(args.prefix, args.instances)

From a946a4dbd8252761cd4f82abdeed12b8c80e3ab7 Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Thu, 29 Nov 2018 19:04:16 -0800
Subject: [PATCH 02/27] fix missing prefix for bring-up.sh

---
 scripts/load_testing/server_config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/load_testing/server_config.py b/scripts/load_testing/server_config.py
index d23d73766..e72619519 100644
--- a/scripts/load_testing/server_config.py
+++ b/scripts/load_testing/server_config.py
@@ -61,7 +61,7 @@ def generate_config(prefix, instances, slab_mem):
 def generate_runscript(prefix, instances):
   config_path = os.path.join(prefix, 'config')
   # create bring-up.sh
-  with open('bring-up.sh','w') as the_file:
+  with open(os.path.join(prefix, 'bring-up.sh'),'w') as the_file:
     for i in range(instances):
       config_file = os.path.join(config_path, 'pelikan-{server_port}.config'.format(server_port=PELIKAN_SERVER_PORT+i))
       if BIND_TO_NODES:

From 5bfd16082f7149d2b96509264dff496dba8b2c4b Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Fri, 30 Nov 2018 16:28:54 -0800
Subject: [PATCH 03/27] fix some bugs wrt file/dir creation

---
 scripts/load_testing/server_config.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/scripts/load_testing/server_config.py b/scripts/load_testing/server_config.py
index e72619519..de2ff0b9f 100644
--- a/scripts/load_testing/server_config.py
+++ b/scripts/load_testing/server_config.py
@@ -61,7 +61,8 @@ def generate_config(prefix, instances, slab_mem):
 def generate_runscript(prefix, instances):
   config_path = os.path.join(prefix, 'config')
   # create bring-up.sh
-  with open(os.path.join(prefix, 'bring-up.sh'),'w') as the_file:
+  fname = os.path.join(prefix, 'bring-up.sh')
+  with open(fname,'w') as the_file:
     for i in range(instances):
       config_file = os.path.join(config_path, 'pelikan-{server_port}.config'.format(server_port=PELIKAN_SERVER_PORT+i))
       if BIND_TO_NODES:
@@ -74,7 +75,7 @@ def generate_runscript(prefix, instances):
       the_file.write('{binary_file} {config_file}\n'.format(
           binary_file=PELIKAN_BINARY,
           config_file=config_file))
-  os.chmod('bring-up.sh', 0777)
+  os.chmod(fname, 0777)
 
 if __name__ == "__main__":
   parser = argparse.ArgumentParser(description="""
@@ -86,5 +87,8 @@ def generate_runscript(prefix, instances):
 
   args = parser.parse_args()
 
+  if not os.path.exists(args.prefix):
+    os.makedirs(args.prefix)
+
   generate_config(args.prefix, args.instances, args.slab_mem)
   generate_runscript(args.prefix, args.instances)

From 84bf421ce4d502fc8c6816482b584b25d18d4f9e Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Fri, 30 Nov 2018 16:29:35 -0800
Subject: [PATCH 04/27] adding client script

---
 scripts/load_testing/client_config.py | 82 +++++++++++++++++++++++++++
 1 file changed, 82 insertions(+)
 create mode 100644 scripts/load_testing/client_config.py

diff --git a/scripts/load_testing/client_config.py b/scripts/load_testing/client_config.py
new file mode 100644
index 000000000..1d23278f0
--- /dev/null
+++ b/scripts/load_testing/client_config.py
@@ -0,0 +1,82 @@
+import argparse
+import os
+
+INSTANCES = 3
+PREFIX = 'test'
+THREADS = 1
+CONNECTIONS = 100
+RATE = 10000
+GET_RATIO = 0.9
+SET_RATIO = 1 - GET_RATIO
+SIZE = 4
+PELIKAN_SERVER_PORT = 12300
+PELIKAN_SERVER_IP = '10.25.2.44'
+RPCPERF_BINARY = '/root/Twitter/rpc-perf/target/release/rpc-perf'
+
+def generate_config(prefix, threads, connections, rate, size):
+# create rpcperf.toml
+  rate_get = int(rate * GET_RATIO)
+  rate_set = int(rate * SET_RATIO)
+  config_str = """\
+[general]
+threads = {threads}
+tcp-nodelay = true
+connections = {connections}
+windows = 600
+duration = 60
+request-timeout = 200
+connect-timeout = 50
+
+[[workload]]
+name = "get"
+method = "get"
+rate = {rate_get}
+  [[workload.parameter]]
+  style = "random"
+  size = {size}
+  regenerate = true
+
+[[workload]]
+name = "set"
+method = "set"
+rate = {rate_set}
+  [[workload.parameter]]
+  style = "random"
+  size = {size}
+  regenerate = true
+[[workload.parameter]]
+  style = "random"
+  size = {size}
+  regenerate = false
+  """.format(threads=threads, connections=connections, rate_get=rate_get, rate_set=rate_set, size=size)
+  with open(os.path.join(prefix, 'rpcperf.toml'), 'w') as the_file:
+    the_file.write(config_str)
+
+def generate_runscript(prefix, instances):
+# create test.sh
+  fname = os.path.join(prefix, 'test.sh')
+  with open(fname, 'w') as the_file:
+    for i in range(instances):
+      server_port = PELIKAN_SERVER_PORT + i
+      the_file.write('{binary_file} --config {config_file}'.format(binary_file=RPCPERF_BINARY, config_file='rpcperf.toml'))
+      the_file.write(' --server {server_ip}:{server_port} &\n'.format(server_ip=PELIKAN_SERVER_IP, server_port=server_port))
+  os.chmod(fname, 0777)
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="""
+    Generate all the client-side scripts/configs needed for a test run.
+    """)
+  parser.add_argument('--prefix', dest='prefix', type=str, default=PREFIX, help='folder that contains all the other files to be generated')
+  parser.add_argument('--instances', dest='instances', type=int, default=INSTANCES, help='number of instances')
+  parser.add_argument('--threads', dest='threads', type=int, default=THREADS, help='number of worker threads per rpc-perf')
+  parser.add_argument('--connections', dest='connections', type=int, default=CONNECTIONS, help='number of connections PER THREAD')
+  parser.add_argument('--rate', dest='rate', type=int, default=RATE, help='aggregated request rate')
+  parser.add_argument('--size', dest='size', type=int, default=SIZE, help='payload size')
+
+  args = parser.parse_args()
+
+  if not os.path.exists(args.prefix):
+    os.makedirs(args.prefix)
+
+  generate_config(args.prefix, args.threads, args.connections, args.rate, args.size)
+  generate_runscript(args.prefix, args.instances)

From ef6d849c62fee381838c8457cbbad86e2c46d242 Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Tue, 11 Dec 2018 19:37:21 -0800
Subject: [PATCH 05/27] add config option for key/val size (which adjusts
 hash_power value)

---
 scripts/load_testing/server_config.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/scripts/load_testing/server_config.py b/scripts/load_testing/server_config.py
index de2ff0b9f..cf90623c1 100644
--- a/scripts/load_testing/server_config.py
+++ b/scripts/load_testing/server_config.py
@@ -1,18 +1,20 @@
-import os
 import argparse
+from math import ceil, floor, log
+import os
 
 INSTANCES = 3
 PREFIX = 'test'
 PELIKAN_ADMIN_PORT = 9900
 PELIKAN_SERVER_PORT = 12300
 PELIKAN_SERVER_IP = '10.25.2.45'
+PELIKAN_SIZE = 64
 PELIKAN_SLAB_MEM = 4294967296
 PELIKAN_BINARY = '/root/Twitter/pelikan/_build/_bin/pelikan_twemcache'
 THREAD_PER_SOCKET = 48
 BIND_TO_CORES = False
 BIND_TO_NODES = True
 
-def generate_config(prefix, instances, slab_mem):
+def generate_config(prefix, instances, hash_power, slab_mem):
   # create top-level folders under prefix
   config_path = os.path.join(prefix, 'config')
   os.makedirs(config_path)
@@ -47,10 +49,10 @@ def generate_config(prefix, instances, slab_mem):
 
 slab_evict_opt: 1
 slab_prealloc: yes
-slab_hash_power: 26
+slab_hash_power: {hash_power}
 slab_mem: {slab_mem}
 slab_size: 1048756
-""".format(admin_port=admin_port, server_port=server_port, slab_mem=slab_mem)
+""".format(admin_port=admin_port, server_port=server_port, hash_power=hash_power, slab_mem=slab_mem)
     try:
       os.makedirs(os.path.join(log_path, str(server_port)))
     except:
@@ -83,12 +85,16 @@ def generate_runscript(prefix, instances):
     """)
   parser.add_argument('--prefix', dest='prefix', type=str, default=PREFIX, help='folder that contains all the other files to be generated')
   parser.add_argument('--instances', dest='instances', type=int, default=INSTANCES, help='number of instances')
+  parser.add_argument('--size', dest='size', type=int, default=PELIKAN_SIZE, help='key+val total size')
   parser.add_argument('--slab_mem', dest='slab_mem', type=int, default=PELIKAN_SLAB_MEM, help='total capacity of slab memory, in bytes')
 
   args = parser.parse_args()
 
+  nkey = 1.0 * args.slab_mem / args.size
+  hash_power = int(ceil(log(nkey, 2)))
+
   if not os.path.exists(args.prefix):
     os.makedirs(args.prefix)
 
-  generate_config(args.prefix, args.instances, args.slab_mem)
+  generate_config(args.prefix, args.instances, hash_power, args.slab_mem)
   generate_runscript(args.prefix, args.instances)

From c02b8d425f03e3f6a91e3ff6f38b912a2b9e4768 Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Wed, 12 Dec 2018 14:59:26 -0800
Subject: [PATCH 06/27] adding bash script to generate server config groups

---
 scripts/load_testing/generate.sh | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100755 scripts/load_testing/generate.sh

diff --git a/scripts/load_testing/generate.sh b/scripts/load_testing/generate.sh
new file mode 100755
index 000000000..6b33567f6
--- /dev/null
+++ b/scripts/load_testing/generate.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+mem_configs=(4 8 16 32)
+size_configs=(32 64 128 256 512 1024 2048)
+instances=30
+
+for size in "${size_configs[@]}"
+do
+    for mem in "${mem_configs[@]}"
+    do
+        slab_mem=$((mem * 1024 * 1024 * 1024))
+        prefix=test_${size}_${mem}
+        python server_config.py --prefix="$prefix" --instances="$instances" --slab_mem "$slab_mem" --size "$size"
+    done
+done

From e4e9c535d7e12a525d13601a5eb3d948f096670c Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Fri, 14 Dec 2018 17:07:10 -0800
Subject: [PATCH 07/27] making config file path relative/local to bring-up.sh

---
 scripts/load_testing/server_config.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/scripts/load_testing/server_config.py b/scripts/load_testing/server_config.py
index cf90623c1..b344b20b8 100644
--- a/scripts/load_testing/server_config.py
+++ b/scripts/load_testing/server_config.py
@@ -7,9 +7,9 @@
 PELIKAN_ADMIN_PORT = 9900
 PELIKAN_SERVER_PORT = 12300
 PELIKAN_SERVER_IP = '10.25.2.45'
-PELIKAN_SIZE = 64
 PELIKAN_SLAB_MEM = 4294967296
 PELIKAN_BINARY = '/root/Twitter/pelikan/_build/_bin/pelikan_twemcache'
+SIZE = 64
 THREAD_PER_SOCKET = 48
 BIND_TO_CORES = False
 BIND_TO_NODES = True
@@ -61,12 +61,11 @@ def generate_config(prefix, instances, hash_power, slab_mem):
       the_file.write(config_str)
 
 def generate_runscript(prefix, instances):
-  config_path = os.path.join(prefix, 'config')
   # create bring-up.sh
   fname = os.path.join(prefix, 'bring-up.sh')
   with open(fname,'w') as the_file:
     for i in range(instances):
-      config_file = os.path.join(config_path, 'pelikan-{server_port}.config'.format(server_port=PELIKAN_SERVER_PORT+i))
+      config_file = os.path.join('config', 'pelikan-{server_port}.config'.format(server_port=PELIKAN_SERVER_PORT+i))
       if BIND_TO_NODES:
         the_file.write('sudo numactl --cpunodebind={numa_node} --preferred={numa_node} '.format(
             numa_node=i%2))
@@ -85,7 +84,7 @@ def generate_runscript(prefix, instances):
     """)
   parser.add_argument('--prefix', dest='prefix', type=str, default=PREFIX, help='folder that contains all the other files to be generated')
   parser.add_argument('--instances', dest='instances', type=int, default=INSTANCES, help='number of instances')
-  parser.add_argument('--size', dest='size', type=int, default=PELIKAN_SIZE, help='key+val total size')
+  parser.add_argument('--size', dest='size', type=int, default=SIZE, help='key+val total size')
   parser.add_argument('--slab_mem', dest='slab_mem', type=int, default=PELIKAN_SLAB_MEM, help='total capacity of slab memory, in bytes')
 
   args = parser.parse_args()

From 409f010d57b665a66573aa74b0912e19f40844f4 Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Fri, 14 Dec 2018 17:58:29 -0800
Subject: [PATCH 08/27] using memcache style timestamp to work with rpcperf
 properly (which assumes that); improve key # calculation accuracy

---
 scripts/load_testing/server_config.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/load_testing/server_config.py b/scripts/load_testing/server_config.py
index b344b20b8..469136523 100644
--- a/scripts/load_testing/server_config.py
+++ b/scripts/load_testing/server_config.py
@@ -8,6 +8,7 @@
 PELIKAN_SERVER_PORT = 12300
 PELIKAN_SERVER_IP = '10.25.2.45'
 PELIKAN_SLAB_MEM = 4294967296
+PELIKAN_ITEM_OVERHEAD = 48
 PELIKAN_BINARY = '/root/Twitter/pelikan/_build/_bin/pelikan_twemcache'
 SIZE = 64
 THREAD_PER_SOCKET = 48
@@ -52,6 +53,8 @@ def generate_config(prefix, instances, hash_power, slab_mem):
 slab_hash_power: {hash_power}
 slab_mem: {slab_mem}
 slab_size: 1048756
+
+time_type: 2
 """.format(admin_port=admin_port, server_port=server_port, hash_power=hash_power, slab_mem=slab_mem)
     try:
       os.makedirs(os.path.join(log_path, str(server_port)))
@@ -89,7 +92,7 @@ def generate_runscript(prefix, instances):
 
   args = parser.parse_args()
 
-  nkey = 1.0 * args.slab_mem / args.size
+  nkey = 1.0 * args.slab_mem / (args.size + PELIKAN_ITEM_OVERHEAD)
   hash_power = int(ceil(log(nkey, 2)))
 
   if not os.path.exists(args.prefix):

From 9f7621c344ff28e95562b7b664c9de8bc7bbd25b Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Tue, 18 Dec 2018 14:00:42 -0800
Subject: [PATCH 09/27] update server-side config generator with prefill
 options and minor fixes

---
 scripts/load_testing/generate.sh      |  8 +++++---
 scripts/load_testing/server_config.py | 23 +++++++++++++++--------
 2 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/scripts/load_testing/generate.sh b/scripts/load_testing/generate.sh
index 6b33567f6..4d01c61eb 100755
--- a/scripts/load_testing/generate.sh
+++ b/scripts/load_testing/generate.sh
@@ -1,15 +1,17 @@
 #!/bin/bash
 
 mem_configs=(4 8 16 32)
-size_configs=(32 64 128 256 512 1024 2048)
+size_configs=(64 128 256 512 1024 2048)
 instances=30
+ksize=32
 
 for size in "${size_configs[@]}"
 do
     for mem in "${mem_configs[@]}"
     do
         slab_mem=$((mem * 1024 * 1024 * 1024))
-        prefix=test_${size}_${mem}
-        python server_config.py --prefix="$prefix" --instances="$instances" --slab_mem "$slab_mem" --size "$size"
+        vsize=$((size - ksize))
+        prefix=pelikan_${size}_${mem}
+        python server_config.py --prefix="$prefix" --instances="$instances" --slab_mem "$slab_mem" --vsize "$vsize"
     done
 done
diff --git a/scripts/load_testing/server_config.py b/scripts/load_testing/server_config.py
index 469136523..d7fb345e7 100644
--- a/scripts/load_testing/server_config.py
+++ b/scripts/load_testing/server_config.py
@@ -10,18 +10,23 @@
 PELIKAN_SLAB_MEM = 4294967296
 PELIKAN_ITEM_OVERHEAD = 48
 PELIKAN_BINARY = '/root/Twitter/pelikan/_build/_bin/pelikan_twemcache'
-SIZE = 64
+KSIZE = 32
+VSIZE = 32
 THREAD_PER_SOCKET = 48
 BIND_TO_CORES = False
 BIND_TO_NODES = True
 
-def generate_config(prefix, instances, hash_power, slab_mem):
+def generate_config(prefix, instances, vsize, slab_mem):
   # create top-level folders under prefix
   config_path = os.path.join(prefix, 'config')
   os.makedirs(config_path)
   log_path = os.path.join(prefix, 'log')
   os.makedirs(log_path)
 
+
+  nkey = int(ceil(1.0 * slab_mem / (vsize + KSIZE + PELIKAN_ITEM_OVERHEAD)))
+  hash_power = int(ceil(log(nkey, 2)))
+
   # create twemcache config file(s)
   for i in range(instances):
     admin_port = PELIKAN_ADMIN_PORT + i
@@ -45,6 +50,11 @@ def generate_config(prefix, instances, hash_power, slab_mem):
 klog_sample: 100
 klog_max: 1073741824
 
+prefill: yes
+prefill_ksize: 32
+prefill_vsize: {vsize}
+prefill_nkey: {nkey}
+
 request_poolsize: 16384
 response_poolsize: 32768
 
@@ -55,7 +65,7 @@ def generate_config(prefix, instances, hash_power, slab_mem):
 slab_size: 1048756
 
 time_type: 2
-""".format(admin_port=admin_port, server_port=server_port, hash_power=hash_power, slab_mem=slab_mem)
+""".format(admin_port=admin_port, server_port=server_port, vsize=vsize, nkey=nkey, hash_power=hash_power, slab_mem=slab_mem)
     try:
       os.makedirs(os.path.join(log_path, str(server_port)))
     except:
@@ -87,16 +97,13 @@ def generate_runscript(prefix, instances):
     """)
   parser.add_argument('--prefix', dest='prefix', type=str, default=PREFIX, help='folder that contains all the other files to be generated')
   parser.add_argument('--instances', dest='instances', type=int, default=INSTANCES, help='number of instances')
-  parser.add_argument('--size', dest='size', type=int, default=SIZE, help='key+val total size')
+  parser.add_argument('--vsize', dest='vsize', type=int, default=VSIZE, help='value size')
   parser.add_argument('--slab_mem', dest='slab_mem', type=int, default=PELIKAN_SLAB_MEM, help='total capacity of slab memory, in bytes')
 
   args = parser.parse_args()
 
-  nkey = 1.0 * args.slab_mem / (args.size + PELIKAN_ITEM_OVERHEAD)
-  hash_power = int(ceil(log(nkey, 2)))
-
   if not os.path.exists(args.prefix):
     os.makedirs(args.prefix)
 
-  generate_config(args.prefix, args.instances, hash_power, args.slab_mem)
+  generate_config(args.prefix, args.instances, args.vsize, args.slab_mem)
   generate_runscript(args.prefix, args.instances)

From 9f791895c494a51eb5c252587c21dc00af72923d Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Tue, 18 Dec 2018 15:30:58 -0800
Subject: [PATCH 10/27] update client_config.py to match (server) prefilling
 pattern

---
 scripts/load_testing/client_config.py | 65 ++++++++++++++++-----------
 scripts/load_testing/generate.sh      | 21 ++++++++-
 scripts/load_testing/server_config.py | 22 +++++----
 3 files changed, 70 insertions(+), 38 deletions(-)

diff --git a/scripts/load_testing/client_config.py b/scripts/load_testing/client_config.py
index 1d23278f0..882bb2d04 100644
--- a/scripts/load_testing/client_config.py
+++ b/scripts/load_testing/client_config.py
@@ -1,22 +1,31 @@
 import argparse
+from math import ceil
 import os
 
+
 INSTANCES = 3
-PREFIX = 'test'
-THREADS = 1
-CONNECTIONS = 100
-RATE = 10000
-GET_RATIO = 0.9
-SET_RATIO = 1 - GET_RATIO
-SIZE = 4
+PREFIX = 'loadgen'
+RPCPERF_THREADS = 1
+RPCPERF_CONNS = 100
+RPCPERF_RATE = 10000
+RPCPERF_GET_RATIO = 0.9
+RPCPERF_SET_RATIO = 1 - RPCPERF_GET_RATIO
+PELIKAN_SLAB_MEM = 4294967296
+PELIKAN_ITEM_OVERHEAD = 48
+KSIZE = 32
+VSIZE = 32
 PELIKAN_SERVER_PORT = 12300
 PELIKAN_SERVER_IP = '10.25.2.44'
 RPCPERF_BINARY = '/root/Twitter/rpc-perf/target/release/rpc-perf'
 
-def generate_config(prefix, threads, connections, rate, size):
+
+def generate_config(rate, connections, vsize, slab_mem, threads):
 # create rpcperf.toml
-  rate_get = int(rate * GET_RATIO)
-  rate_set = int(rate * SET_RATIO)
+  rate_get = int(rate * RPCPERF_GET_RATIO)
+  rate_set = int(rate * RPCPERF_SET_RATIO)
+  nkey = int(ceil(1.0 * slab_mem / (vsize + KSIZE + PELIKAN_ITEM_OVERHEAD)))
+  conn_per_thread = connections / threads
+
   config_str = """\
 [general]
 threads = {threads}
@@ -33,7 +42,8 @@ def generate_config(prefix, threads, connections, rate, size):
 rate = {rate_get}
   [[workload.parameter]]
   style = "random"
-  size = {size}
+  size = 32
+  num = {nkey}
   regenerate = true
 
 [[workload]]
@@ -42,19 +52,21 @@ def generate_config(prefix, threads, connections, rate, size):
 rate = {rate_set}
   [[workload.parameter]]
   style = "random"
-  size = {size}
+  size = 32
+  num = {nkey}
   regenerate = true
-[[workload.parameter]]
+  [[workload.parameter]]
   style = "random"
-  size = {size}
+  size = {vsize}
   regenerate = false
-  """.format(threads=threads, connections=connections, rate_get=rate_get, rate_set=rate_set, size=size)
-  with open(os.path.join(prefix, 'rpcperf.toml'), 'w') as the_file:
+""".format(threads=threads, connections=conn_per_thread, nkey=nkey, rate_get=rate_get, rate_set=rate_set, vsize=vsize)
+  with open('rpcperf.toml', 'w') as the_file:
     the_file.write(config_str)
 
-def generate_runscript(prefix, instances):
-# create test.sh
-  fname = os.path.join(prefix, 'test.sh')
+
+def generate_runscript(instances):
+  # create test.sh
+  fname = 'test.sh'
   with open(fname, 'w') as the_file:
     for i in range(instances):
       server_port = PELIKAN_SERVER_PORT + i
@@ -62,21 +74,24 @@ def generate_runscript(prefix, instances):
       the_file.write(' --server {server_ip}:{server_port} &\n'.format(server_ip=PELIKAN_SERVER_IP, server_port=server_port))
   os.chmod(fname, 0777)
 
+
 if __name__ == "__main__":
   parser = argparse.ArgumentParser(description="""
     Generate all the client-side scripts/configs needed for a test run.
     """)
   parser.add_argument('--prefix', dest='prefix', type=str, default=PREFIX, help='folder that contains all the other files to be generated')
   parser.add_argument('--instances', dest='instances', type=int, default=INSTANCES, help='number of instances')
-  parser.add_argument('--threads', dest='threads', type=int, default=THREADS, help='number of worker threads per rpc-perf')
-  parser.add_argument('--connections', dest='connections', type=int, default=CONNECTIONS, help='number of connections PER THREAD')
-  parser.add_argument('--rate', dest='rate', type=int, default=RATE, help='aggregated request rate')
-  parser.add_argument('--size', dest='size', type=int, default=SIZE, help='payload size')
+  parser.add_argument('--rate', dest='rate', type=int, default=RPCPERF_RATE, help='request rate per instance')
+  parser.add_argument('--connections', dest='connections', type=int, default=RPCPERF_CONNS, help='number of connections per instance')
+  parser.add_argument('--vsize', dest='vsize', type=int, default=VSIZE, help='value size')
+  parser.add_argument('--slab_mem', dest='slab_mem', type=int, default=PELIKAN_SLAB_MEM, help='slab memory')
+  parser.add_argument('--threads', dest='threads', type=int, default=RPCPERF_THREADS, help='number of worker threads per rpc-perf')
 
   args = parser.parse_args()
 
   if not os.path.exists(args.prefix):
     os.makedirs(args.prefix)
+  os.chdir(args.prefix)
 
-  generate_config(args.prefix, args.threads, args.connections, args.rate, args.size)
-  generate_runscript(args.prefix, args.instances)
+  generate_config(args.rate, args.connections, args.vsize, args.slab_mem, args.threads)
+  generate_runscript(args.instances)
diff --git a/scripts/load_testing/generate.sh b/scripts/load_testing/generate.sh
index 4d01c61eb..3113af686 100755
--- a/scripts/load_testing/generate.sh
+++ b/scripts/load_testing/generate.sh
@@ -1,17 +1,36 @@
 #!/bin/bash
 
+conn_configs=(100 1000 10000)
 mem_configs=(4 8 16 32)
 size_configs=(64 128 256 512 1024 2048)
 instances=30
 ksize=32
+rate=100000
+threads=2
 
+# pelikan configs
 for size in "${size_configs[@]}"
 do
+    vsize=$((size - ksize))
     for mem in "${mem_configs[@]}"
     do
         slab_mem=$((mem * 1024 * 1024 * 1024))
-        vsize=$((size - ksize))
         prefix=pelikan_${size}_${mem}
         python server_config.py --prefix="$prefix" --instances="$instances" --slab_mem "$slab_mem" --vsize "$vsize"
     done
 done
+
+# rpc-perf configs
+for conn in "${conn_configs[@]}"
+do
+    for size in "${size_configs[@]}"
+    do
+        vsize=$((size - ksize))
+        for mem in "${mem_configs[@]}"
+        do
+            slab_mem=$((mem * 1024 * 1024 * 1024))
+            prefix=rpcperf_${conn}_${size}_${mem}
+            python client_config.py --prefix="$prefix" --instances="$instances" --rate="$rate" --connections="$conn" --vsize "$vsize" --slab_mem="$slab_mem" --threads="$threads"
+        done
+    done
+done
diff --git a/scripts/load_testing/server_config.py b/scripts/load_testing/server_config.py
index d7fb345e7..429b3e4bb 100644
--- a/scripts/load_testing/server_config.py
+++ b/scripts/load_testing/server_config.py
@@ -16,13 +16,10 @@
 BIND_TO_CORES = False
 BIND_TO_NODES = True
 
-def generate_config(prefix, instances, vsize, slab_mem):
+def generate_config(instances, vsize, slab_mem):
   # create top-level folders under prefix
-  config_path = os.path.join(prefix, 'config')
-  os.makedirs(config_path)
-  log_path = os.path.join(prefix, 'log')
-  os.makedirs(log_path)
-
+  os.makedirs('config')
+  os.makedirs('log')
 
   nkey = int(ceil(1.0 * slab_mem / (vsize + KSIZE + PELIKAN_ITEM_OVERHEAD)))
   hash_power = int(ceil(log(nkey, 2)))
@@ -67,15 +64,15 @@ def generate_config(prefix, instances, vsize, slab_mem):
 time_type: 2
 """.format(admin_port=admin_port, server_port=server_port, vsize=vsize, nkey=nkey, hash_power=hash_power, slab_mem=slab_mem)
     try:
-      os.makedirs(os.path.join(log_path, str(server_port)))
+      os.makedirs(os.path.join('log', str(server_port)))
     except:
       pass
-    with open(os.path.join(config_path, config_file),'w') as the_file:
+    with open(os.path.join('config', config_file),'w') as the_file:
       the_file.write(config_str)
 
-def generate_runscript(prefix, instances):
+def generate_runscript(instances):
   # create bring-up.sh
-  fname = os.path.join(prefix, 'bring-up.sh')
+  fname = 'bring-up.sh'
   with open(fname,'w') as the_file:
     for i in range(instances):
       config_file = os.path.join('config', 'pelikan-{server_port}.config'.format(server_port=PELIKAN_SERVER_PORT+i))
@@ -104,6 +101,7 @@ def generate_runscript(prefix, instances):
 
   if not os.path.exists(args.prefix):
     os.makedirs(args.prefix)
+  os.chdir(args.prefix)
 
-  generate_config(args.prefix, args.instances, args.vsize, args.slab_mem)
-  generate_runscript(args.prefix, args.instances)
+  generate_config(args.instances, args.vsize, args.slab_mem)
+  generate_runscript(args.instances)

From 91abd183da62405e96592fdddcbe447601f54c83 Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Tue, 18 Dec 2018 17:10:01 -0800
Subject: [PATCH 11/27] update client config parameters per Brian's suggestion

---
 scripts/load_testing/client_config.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/load_testing/client_config.py b/scripts/load_testing/client_config.py
index 882bb2d04..381b79413 100644
--- a/scripts/load_testing/client_config.py
+++ b/scripts/load_testing/client_config.py
@@ -31,10 +31,10 @@ def generate_config(rate, connections, vsize, slab_mem, threads):
 threads = {threads}
 tcp-nodelay = true
 connections = {connections}
-windows = 600
+windows = 10
 duration = 60
 request-timeout = 200
-connect-timeout = 50
+connect-timeout = 250
 
 [[workload]]
 name = "get"

From 2c4ca06c434dd0d3ed3ffd6360d1663e36d5180e Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Tue, 18 Dec 2018 17:23:32 -0800
Subject: [PATCH 12/27] improve client config; add comment; add waterfall chart
 output; add log output

---
 scripts/load_testing/client_config.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/scripts/load_testing/client_config.py b/scripts/load_testing/client_config.py
index 381b79413..a6b70fe46 100644
--- a/scripts/load_testing/client_config.py
+++ b/scripts/load_testing/client_config.py
@@ -30,9 +30,10 @@ def generate_config(rate, connections, vsize, slab_mem, threads):
 [general]
 threads = {threads}
 tcp-nodelay = true
-connections = {connections}
-windows = 10
-duration = 60
+connections = {connections} # this specifies number of connection per thread
+# runtime ~= windows x duration
+windows = 1
+duration = 600
 request-timeout = 200
 connect-timeout = 250
 
@@ -50,12 +51,12 @@ def generate_config(rate, connections, vsize, slab_mem, threads):
 name = "set"
 method = "set"
 rate = {rate_set}
-  [[workload.parameter]]
+  [[workload.parameter]] # key generation parameters
   style = "random"
   size = 32
   num = {nkey}
   regenerate = true
-  [[workload.parameter]]
+  [[workload.parameter]] # value generation parameters
   style = "random"
   size = {vsize}
   regenerate = false
@@ -71,7 +72,11 @@ def generate_runscript(instances):
     for i in range(instances):
       server_port = PELIKAN_SERVER_PORT + i
       the_file.write('{binary_file} --config {config_file}'.format(binary_file=RPCPERF_BINARY, config_file='rpcperf.toml'))
-      the_file.write(' --server {server_ip}:{server_port} &\n'.format(server_ip=PELIKAN_SERVER_IP, server_port=server_port))
+      the_file.write(' --server {server_ip}:{server_port}'.format(server_ip=PELIKAN_SERVER_IP, server_port=server_port))
+      the_file.write(' --waterfall latency-waterfall-{server_port}.png'.format(server_port=server_port))
+      the_file.write(' 2>&1')
+      the_file.write(' > rpcperf_{server_port}.log'.format(server_port=server_port))
+      the_file.write(' &\n')
   os.chmod(fname, 0777)
 
 

From 95f68225ca54bfdcab79e3d772a94df73bbfe298 Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Wed, 19 Dec 2018 16:45:04 -0800
Subject: [PATCH 13/27] flattern log dir config for stacked server jobs

---
 scripts/load_testing/server_config.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/scripts/load_testing/server_config.py b/scripts/load_testing/server_config.py
index 429b3e4bb..4ae23cbf1 100644
--- a/scripts/load_testing/server_config.py
+++ b/scripts/load_testing/server_config.py
@@ -18,8 +18,14 @@
 
 def generate_config(instances, vsize, slab_mem):
   # create top-level folders under prefix
-  os.makedirs('config')
-  os.makedirs('log')
+  try:
+    os.makedirs('config')
+  except:
+    pass
+  try:
+    os.makedirs('log')
+  except:
+    pass
 
   nkey = int(ceil(1.0 * slab_mem / (vsize + KSIZE + PELIKAN_ITEM_OVERHEAD)))
   hash_power = int(ceil(log(nkey, 2)))
@@ -28,7 +34,7 @@ def generate_config(instances, vsize, slab_mem):
   for i in range(instances):
     admin_port = PELIKAN_ADMIN_PORT + i
     server_port = PELIKAN_SERVER_PORT + i
-    config_file = 'pelikan-{server_port}.config'.format(server_port=server_port)
+    config_file = 'twemcache-{server_port}.config'.format(server_port=server_port)
     config_str = """\
 daemonize: yes
 admin_port: {admin_port}
@@ -39,11 +45,11 @@ def generate_config(instances, vsize, slab_mem):
 buf_sock_poolsize: 16384
 
 debug_log_level: 5
-debug_log_file: log/{server_port}/twemcache.log
+debug_log_file: log/twemcache-{server_port}.log
 debug_log_nbuf: 1048576
 
-klog_file: log/{server_port}/twemcache.cmd
-klog_backup: log/{server_port}/twemcache.cmd.old
+klog_file: log/twemcache-{server_port}.cmd
+klog_backup: log/twemcache-{server_port}.cmd.old
 klog_sample: 100
 klog_max: 1073741824
 
@@ -63,10 +69,6 @@ def generate_config(instances, vsize, slab_mem):
 
 time_type: 2
 """.format(admin_port=admin_port, server_port=server_port, vsize=vsize, nkey=nkey, hash_power=hash_power, slab_mem=slab_mem)
-    try:
-      os.makedirs(os.path.join('log', str(server_port)))
-    except:
-      pass
     with open(os.path.join('config', config_file),'w') as the_file:
       the_file.write(config_str)
 
@@ -75,7 +77,7 @@ def generate_runscript(instances):
   fname = 'bring-up.sh'
   with open(fname,'w') as the_file:
     for i in range(instances):
-      config_file = os.path.join('config', 'pelikan-{server_port}.config'.format(server_port=PELIKAN_SERVER_PORT+i))
+      config_file = os.path.join('config', 'twemcache-{server_port}.config'.format(server_port=PELIKAN_SERVER_PORT+i))
       if BIND_TO_NODES:
         the_file.write('sudo numactl --cpunodebind={numa_node} --preferred={numa_node} '.format(
             numa_node=i%2))

From ac89b1a1a2d6b0c73c33d0257328409880c19202 Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Wed, 19 Dec 2018 18:26:03 -0800
Subject: [PATCH 14/27] add script to launch a single test

---
 scripts/load_testing/runtest.sh | 81 +++++++++++++++++++++++++++++++++
 1 file changed, 81 insertions(+)
 create mode 100755 scripts/load_testing/runtest.sh

diff --git a/scripts/load_testing/runtest.sh b/scripts/load_testing/runtest.sh
new file mode 100755
index 000000000..a30a7129a
--- /dev/null
+++ b/scripts/load_testing/runtest.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+
+# NOTE(!!): the script only works when all config folders are freshly created,
+# i.e. no data from previous runs
+
+instances=30
+
+# Initialize our own variables:
+client_config=""
+server_config=""
+
+show_help()
+{
+    echo "runtest.sh -c <client_config_path> -s <server_config_path>"
+}
+
+get_args()
+{
+    while getopts ":c:s:h" opt; do
+        case "$opt" in
+        c)  client_config=$OPTARG
+            ;;
+        s)  server_config=$OPTARG
+            ;;
+        h)
+            show_help
+            exit 0
+            ;;
+        \?)
+            echo "unrecognized option $opt"
+            show_help
+            exit 1
+            ;;
+        esac
+    done
+}
+
+server_warmup()
+{
+    cd "$server_config" || exit 1
+
+    ./bring-up.sh
+
+    local nready=0
+    while [ $nready -lt $instances ]
+    do
+        nready=$(grep -l "prefilling slab" log/twemcache-*.log | wc -l)
+        echo "$(date): $nready out of $instances servers are warmed up"
+        sleep 10
+    done
+
+    cd - > /dev/null || exit 1
+}
+
+
+client_run()
+{
+    cd "$client_config" || exit 1
+
+    ./test.sh
+
+    local nrunning=1
+    while [ $nrunning -gt 0 ]
+    do
+        nrunning=$(pgrep -c rpc-perf)
+        echo "$(date): $nrunning clients are still running"
+        sleep 10
+    done
+
+    cd - > /dev/null || exit 1
+}
+
+wrap_up()
+{
+    pkill -f pelikan_twemcache
+}
+
+get_args "${@}"
+server_warmup
+client_run
+wrap_up

From 102623c89e72b463d6e6b4bf6c42e0ba7778f85f Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Wed, 19 Dec 2018 19:32:51 -0800
Subject: [PATCH 15/27] make changes needed to launch runtest.sh from the
 client host

---
 scripts/load_testing/client_config.py |  3 +--
 scripts/load_testing/runtest.sh       | 31 +++++++++------------------
 scripts/load_testing/server_config.py | 19 +++++++++++++++-
 3 files changed, 29 insertions(+), 24 deletions(-)

diff --git a/scripts/load_testing/client_config.py b/scripts/load_testing/client_config.py
index a6b70fe46..2fe67e409 100644
--- a/scripts/load_testing/client_config.py
+++ b/scripts/load_testing/client_config.py
@@ -74,9 +74,8 @@ def generate_runscript(instances):
       the_file.write('{binary_file} --config {config_file}'.format(binary_file=RPCPERF_BINARY, config_file='rpcperf.toml'))
       the_file.write(' --server {server_ip}:{server_port}'.format(server_ip=PELIKAN_SERVER_IP, server_port=server_port))
       the_file.write(' --waterfall latency-waterfall-{server_port}.png'.format(server_port=server_port))
-      the_file.write(' 2>&1')
       the_file.write(' > rpcperf_{server_port}.log'.format(server_port=server_port))
-      the_file.write(' &\n')
+      the_file.write(' 2>&1 &\n')
   os.chmod(fname, 0777)
 
 
diff --git a/scripts/load_testing/runtest.sh b/scripts/load_testing/runtest.sh
index a30a7129a..5258e6603 100755
--- a/scripts/load_testing/runtest.sh
+++ b/scripts/load_testing/runtest.sh
@@ -3,25 +3,26 @@
 # NOTE(!!): the script only works when all config folders are freshly created,
 # i.e. no data from previous runs
 
-instances=30
-
 # Initialize our own variables:
 client_config=""
 server_config=""
+target=""
 
 show_help()
 {
-    echo "runtest.sh -c <client_config_path> -s <server_config_path>"
+    echo "runtest.sh -c <client_config_path> -s <server_config_path> -t <target: host where servers run>"
 }
 
 get_args()
 {
-    while getopts ":c:s:h" opt; do
+    while getopts ":c:s:t:h" opt; do
         case "$opt" in
         c)  client_config=$OPTARG
             ;;
         s)  server_config=$OPTARG
             ;;
+        t)  target=$OPTARG
+            ;;
         h)
             show_help
             exit 0
@@ -35,21 +36,9 @@ get_args()
     done
 }
 
-server_warmup()
+server_launch()
 {
-    cd "$server_config" || exit 1
-
-    ./bring-up.sh
-
-    local nready=0
-    while [ $nready -lt $instances ]
-    do
-        nready=$(grep -l "prefilling slab" log/twemcache-*.log | wc -l)
-        echo "$(date): $nready out of $instances servers are warmed up"
-        sleep 10
-    done
-
-    cd - > /dev/null || exit 1
+    ssh -C "$target" "cd $server_config && ./warm-up.sh"
 }
 
 
@@ -72,10 +61,10 @@ client_run()
 
 wrap_up()
 {
-    pkill -f pelikan_twemcache
+    ssh -C "$target" "pkill -f pelikan_twemcache"
 }
 
 get_args "${@}"
-server_warmup
-client_run
+server_launch
+#client_run
 wrap_up
diff --git a/scripts/load_testing/server_config.py b/scripts/load_testing/server_config.py
index 4ae23cbf1..954158ddb 100644
--- a/scripts/load_testing/server_config.py
+++ b/scripts/load_testing/server_config.py
@@ -75,7 +75,7 @@ def generate_config(instances, vsize, slab_mem):
 def generate_runscript(instances):
   # create bring-up.sh
   fname = 'bring-up.sh'
-  with open(fname,'w') as the_file:
+  with open(fname, 'w') as the_file:
     for i in range(instances):
       config_file = os.path.join('config', 'twemcache-{server_port}.config'.format(server_port=PELIKAN_SERVER_PORT+i))
       if BIND_TO_NODES:
@@ -90,6 +90,23 @@ def generate_runscript(instances):
           config_file=config_file))
   os.chmod(fname, 0777)
 
+  # create warm-up.sh
+  fname = 'warm-up.sh'
+  with open(fname, 'w') as the_file:
+    the_file.write("""
+./bring-up.sh
+
+nready=0
+while [ $nready -lt {instances} ]
+do
+    nready=$(grep -l "prefilling slab" log/twemcache-*.log | wc -l)
+    echo "$(date): $nready out of {instances} servers are warmed up"
+    sleep 10
+done
+""".format(instances=instances))
+  os.chmod(fname, 0777)
+
+
 if __name__ == "__main__":
   parser = argparse.ArgumentParser(description="""
     Generate all the server-side scripts/configs needed for a test run.

From 7e121a5ec0a4cbf2190d0ca0f2c17eff968e9fb5 Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Wed, 19 Dec 2018 19:39:05 -0800
Subject: [PATCH 16/27] uncomment client_run

---
 scripts/load_testing/runtest.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/load_testing/runtest.sh b/scripts/load_testing/runtest.sh
index 5258e6603..d5d63ad63 100755
--- a/scripts/load_testing/runtest.sh
+++ b/scripts/load_testing/runtest.sh
@@ -66,5 +66,5 @@ wrap_up()
 
 get_args "${@}"
 server_launch
-#client_run
+client_run
 wrap_up

From 1579a8eaf4c70be06342d51a6cae58e1bad59109 Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Mon, 14 Jan 2019 16:05:12 -0800
Subject: [PATCH 17/27] make binary & server_ip required parameters

---
 scripts/load_testing/client_config.py |  8 +--
 scripts/load_testing/generate.sh      | 83 +++++++++++++++++++++------
 scripts/load_testing/server_config.py |  9 ++-
 3 files changed, 75 insertions(+), 25 deletions(-)

diff --git a/scripts/load_testing/client_config.py b/scripts/load_testing/client_config.py
index 2fe67e409..75af08f22 100644
--- a/scripts/load_testing/client_config.py
+++ b/scripts/load_testing/client_config.py
@@ -15,7 +15,6 @@
 KSIZE = 32
 VSIZE = 32
 PELIKAN_SERVER_PORT = 12300
-PELIKAN_SERVER_IP = '10.25.2.44'
 RPCPERF_BINARY = '/root/Twitter/rpc-perf/target/release/rpc-perf'
 
 
@@ -65,14 +64,14 @@ def generate_config(rate, connections, vsize, slab_mem, threads):
     the_file.write(config_str)
 
 
-def generate_runscript(instances):
+def generate_runscript(server_ip, instances):
   # create test.sh
   fname = 'test.sh'
   with open(fname, 'w') as the_file:
     for i in range(instances):
       server_port = PELIKAN_SERVER_PORT + i
       the_file.write('{binary_file} --config {config_file}'.format(binary_file=RPCPERF_BINARY, config_file='rpcperf.toml'))
-      the_file.write(' --server {server_ip}:{server_port}'.format(server_ip=PELIKAN_SERVER_IP, server_port=server_port))
+      the_file.write(' --server {server_ip}:{server_port}'.format(server_ip=server_ip, server_port=server_port))
       the_file.write(' --waterfall latency-waterfall-{server_port}.png'.format(server_port=server_port))
       the_file.write(' > rpcperf_{server_port}.log'.format(server_port=server_port))
       the_file.write(' 2>&1 &\n')
@@ -85,6 +84,7 @@ def generate_runscript(instances):
     """)
   parser.add_argument('--prefix', dest='prefix', type=str, default=PREFIX, help='folder that contains all the other files to be generated')
   parser.add_argument('--instances', dest='instances', type=int, default=INSTANCES, help='number of instances')
+  parser.add_argument('--server_ip', dest='server_ip', type=str, help='server ip', required=True)
   parser.add_argument('--rate', dest='rate', type=int, default=RPCPERF_RATE, help='request rate per instance')
   parser.add_argument('--connections', dest='connections', type=int, default=RPCPERF_CONNS, help='number of connections per instance')
   parser.add_argument('--vsize', dest='vsize', type=int, default=VSIZE, help='value size')
@@ -98,4 +98,4 @@ def generate_runscript(instances):
   os.chdir(args.prefix)
 
   generate_config(args.rate, args.connections, args.vsize, args.slab_mem, args.threads)
-  generate_runscript(args.instances)
+  generate_runscript(args.server_ip, args.instances)
diff --git a/scripts/load_testing/generate.sh b/scripts/load_testing/generate.sh
index 3113af686..55bb66a39 100755
--- a/scripts/load_testing/generate.sh
+++ b/scripts/load_testing/generate.sh
@@ -8,29 +8,80 @@ ksize=32
 rate=100000
 threads=2
 
-# pelikan configs
-for size in "${size_configs[@]}"
-do
-    vsize=$((size - ksize))
-    for mem in "${mem_configs[@]}"
-    do
-        slab_mem=$((mem * 1024 * 1024 * 1024))
-        prefix=pelikan_${size}_${mem}
-        python server_config.py --prefix="$prefix" --instances="$instances" --slab_mem "$slab_mem" --vsize "$vsize"
+# Initialize our own variables:
+client=false
+server=false
+binary="pelikan_twemcache"
+target="127.0.0.1"
+
+show_help()
+{
+    echo "generate.sh [-c [-t target/serverIP]] [-s [-b binary]]"
+}
+
+get_args()
+{
+    while getopts ":b:t:csh" opt; do
+        case "$opt" in
+        c)  client=true
+            ;;
+        s)  server=true
+            ;;
+        t)  binary=$OPTARG
+            ;;
+        t)  target=$OPTARG
+            ;;
+        h)
+            show_help
+            exit 0
+            ;;
+        \?)
+            echo "unrecognized option $opt"
+            show_help
+            exit 1
+            ;;
+        esac
     done
-done
+}
 
-# rpc-perf configs
-for conn in "${conn_configs[@]}"
-do
+# pelikan configs
+gen_pelikan()
+{
     for size in "${size_configs[@]}"
     do
         vsize=$((size - ksize))
         for mem in "${mem_configs[@]}"
         do
             slab_mem=$((mem * 1024 * 1024 * 1024))
-            prefix=rpcperf_${conn}_${size}_${mem}
-            python client_config.py --prefix="$prefix" --instances="$instances" --rate="$rate" --connections="$conn" --vsize "$vsize" --slab_mem="$slab_mem" --threads="$threads"
+            prefix=pelikan_${size}_${mem}
+            python server_config.py --prefix="$prefix" --binary="$binary" --instances="$instances" --slab_mem "$slab_mem" --vsize "$vsize"
+        done
+    done
+}
+
+# rpc-perf configs
+gen_rpcperf()
+{
+    for conn in "${conn_configs[@]}"
+    do
+        for size in "${size_configs[@]}"
+        do
+            vsize=$((size - ksize))
+            for mem in "${mem_configs[@]}"
+            do
+                slab_mem=$((mem * 1024 * 1024 * 1024))
+                prefix=rpcperf_${conn}_${size}_${mem}
+                python client_config.py --prefix="$prefix" --server_ip="$target" --instances="$instances" --rate="$rate" --connections="$conn" --vsize "$vsize" --slab_mem="$slab_mem" --threads="$threads"
+            done
         done
     done
-done
+}
+
+get_args "${@}"
+if [ "$client" = true ]; then
+    gen_rpcperf
+fi
+if [ "$server" = true ]; then
+    gen_pelikan
+fi
+
diff --git a/scripts/load_testing/server_config.py b/scripts/load_testing/server_config.py
index 954158ddb..ffefbbf56 100644
--- a/scripts/load_testing/server_config.py
+++ b/scripts/load_testing/server_config.py
@@ -6,10 +6,8 @@
 PREFIX = 'test'
 PELIKAN_ADMIN_PORT = 9900
 PELIKAN_SERVER_PORT = 12300
-PELIKAN_SERVER_IP = '10.25.2.45'
 PELIKAN_SLAB_MEM = 4294967296
 PELIKAN_ITEM_OVERHEAD = 48
-PELIKAN_BINARY = '/root/Twitter/pelikan/_build/_bin/pelikan_twemcache'
 KSIZE = 32
 VSIZE = 32
 THREAD_PER_SOCKET = 48
@@ -72,7 +70,7 @@ def generate_config(instances, vsize, slab_mem):
     with open(os.path.join('config', config_file),'w') as the_file:
       the_file.write(config_str)
 
-def generate_runscript(instances):
+def generate_runscript(binary, instances):
   # create bring-up.sh
   fname = 'bring-up.sh'
   with open(fname, 'w') as the_file:
@@ -86,7 +84,7 @@ def generate_runscript(instances):
             physical_thread=i,
             logical_thread=i+THREAD_PER_SOCKET))
       the_file.write('{binary_file} {config_file}\n'.format(
-          binary_file=PELIKAN_BINARY,
+          binary_file=binary,
           config_file=config_file))
   os.chmod(fname, 0777)
 
@@ -111,6 +109,7 @@ def generate_runscript(instances):
   parser = argparse.ArgumentParser(description="""
     Generate all the server-side scripts/configs needed for a test run.
     """)
+  parser.add_argument('--binary', dest='binary', type=str, help='location of pelikan_twemcache binary', required=True)
   parser.add_argument('--prefix', dest='prefix', type=str, default=PREFIX, help='folder that contains all the other files to be generated')
   parser.add_argument('--instances', dest='instances', type=int, default=INSTANCES, help='number of instances')
   parser.add_argument('--vsize', dest='vsize', type=int, default=VSIZE, help='value size')
@@ -123,4 +122,4 @@ def generate_runscript(instances):
   os.chdir(args.prefix)
 
   generate_config(args.instances, args.vsize, args.slab_mem)
-  generate_runscript(args.instances)
+  generate_runscript(args.binary, args.instances)

From 77adec22f5cc0c1ab4470194e6d65ded72c2786a Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Mon, 14 Jan 2019 16:11:26 -0800
Subject: [PATCH 18/27] fix typo

---
 scripts/load_testing/generate.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/load_testing/generate.sh b/scripts/load_testing/generate.sh
index 55bb66a39..7890b528c 100755
--- a/scripts/load_testing/generate.sh
+++ b/scripts/load_testing/generate.sh
@@ -27,7 +27,7 @@ get_args()
             ;;
         s)  server=true
             ;;
-        t)  binary=$OPTARG
+        b)  binary=$OPTARG
             ;;
         t)  target=$OPTARG
             ;;

From 53da8228e905610fec3fb695c97577a6a66b1e0a Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Mon, 14 Jan 2019 17:02:38 -0800
Subject: [PATCH 19/27] make rpc-perf binary location configurable too

---
 scripts/load_testing/client_config.py |  8 ++++----
 scripts/load_testing/generate.sh      | 15 +++++++++------
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/scripts/load_testing/client_config.py b/scripts/load_testing/client_config.py
index 75af08f22..4893c266c 100644
--- a/scripts/load_testing/client_config.py
+++ b/scripts/load_testing/client_config.py
@@ -15,7 +15,6 @@
 KSIZE = 32
 VSIZE = 32
 PELIKAN_SERVER_PORT = 12300
-RPCPERF_BINARY = '/root/Twitter/rpc-perf/target/release/rpc-perf'
 
 
 def generate_config(rate, connections, vsize, slab_mem, threads):
@@ -64,13 +63,13 @@ def generate_config(rate, connections, vsize, slab_mem, threads):
     the_file.write(config_str)
 
 
-def generate_runscript(server_ip, instances):
+def generate_runscript(binary, server_ip, instances):
   # create test.sh
   fname = 'test.sh'
   with open(fname, 'w') as the_file:
     for i in range(instances):
       server_port = PELIKAN_SERVER_PORT + i
-      the_file.write('{binary_file} --config {config_file}'.format(binary_file=RPCPERF_BINARY, config_file='rpcperf.toml'))
+      the_file.write('{binary_file} --config {config_file}'.format(binary_file=binary, config_file='rpcperf.toml'))
       the_file.write(' --server {server_ip}:{server_port}'.format(server_ip=server_ip, server_port=server_port))
       the_file.write(' --waterfall latency-waterfall-{server_port}.png'.format(server_port=server_port))
       the_file.write(' > rpcperf_{server_port}.log'.format(server_port=server_port))
@@ -82,6 +81,7 @@ def generate_runscript(server_ip, instances):
   parser = argparse.ArgumentParser(description="""
     Generate all the client-side scripts/configs needed for a test run.
     """)
+  parser.add_argument('--binary', dest='binary', type=str, help='location of rpc-perf binary', required=True)
   parser.add_argument('--prefix', dest='prefix', type=str, default=PREFIX, help='folder that contains all the other files to be generated')
   parser.add_argument('--instances', dest='instances', type=int, default=INSTANCES, help='number of instances')
   parser.add_argument('--server_ip', dest='server_ip', type=str, help='server ip', required=True)
@@ -98,4 +98,4 @@ def generate_runscript(server_ip, instances):
   os.chdir(args.prefix)
 
   generate_config(args.rate, args.connections, args.vsize, args.slab_mem, args.threads)
-  generate_runscript(args.server_ip, args.instances)
+  generate_runscript(args.binary, args.server_ip, args.instances)
diff --git a/scripts/load_testing/generate.sh b/scripts/load_testing/generate.sh
index 7890b528c..f68a17572 100755
--- a/scripts/load_testing/generate.sh
+++ b/scripts/load_testing/generate.sh
@@ -11,23 +11,26 @@ threads=2
 # Initialize our own variables:
 client=false
 server=false
-binary="pelikan_twemcache"
+rpcperf="rpc-perf"
+pelikan="pelikan_twemcache"
 target="127.0.0.1"
 
 show_help()
 {
-    echo "generate.sh [-c [-t target/serverIP]] [-s [-b binary]]"
+    echo "generate.sh [-c [-r path/to/rpcperf] [-t target/serverIP]] [-s [-p path/to/pelikan]]"
 }
 
 get_args()
 {
-    while getopts ":b:t:csh" opt; do
+    while getopts ":p:r:t:csh" opt; do
         case "$opt" in
         c)  client=true
             ;;
         s)  server=true
             ;;
-        b)  binary=$OPTARG
+        p)  pelikan=$OPTARG
+            ;;
+        r)  rpcperf=$OPTARG
             ;;
         t)  target=$OPTARG
             ;;
@@ -54,7 +57,7 @@ gen_pelikan()
         do
             slab_mem=$((mem * 1024 * 1024 * 1024))
             prefix=pelikan_${size}_${mem}
-            python server_config.py --prefix="$prefix" --binary="$binary" --instances="$instances" --slab_mem "$slab_mem" --vsize "$vsize"
+            python server_config.py --prefix="$prefix" --binary="$pelikan" --instances="$instances" --slab_mem "$slab_mem" --vsize "$vsize"
         done
     done
 }
@@ -71,7 +74,7 @@ gen_rpcperf()
             do
                 slab_mem=$((mem * 1024 * 1024 * 1024))
                 prefix=rpcperf_${conn}_${size}_${mem}
-                python client_config.py --prefix="$prefix" --server_ip="$target" --instances="$instances" --rate="$rate" --connections="$conn" --vsize "$vsize" --slab_mem="$slab_mem" --threads="$threads"
+                python client_config.py --prefix="$prefix" --binary="$rpcperf" --server_ip="$target" --instances="$instances" --rate="$rate" --connections="$conn" --vsize "$vsize" --slab_mem="$slab_mem" --threads="$threads"
             done
         done
     done

From aa35a9696e057af38662381dc0bcdb083a4ad198 Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Tue, 15 Jan 2019 12:43:03 -0800
Subject: [PATCH 20/27] adapt client config for new rpc-perf which has
 different config syntax

---
 scripts/load_testing/client_config.py | 53 ++++++++++-----------------
 1 file changed, 20 insertions(+), 33 deletions(-)

diff --git a/scripts/load_testing/client_config.py b/scripts/load_testing/client_config.py
index 4893c266c..09cc63fa7 100644
--- a/scripts/load_testing/client_config.py
+++ b/scripts/load_testing/client_config.py
@@ -19,46 +19,31 @@
 
 def generate_config(rate, connections, vsize, slab_mem, threads):
 # create rpcperf.toml
-  rate_get = int(rate * RPCPERF_GET_RATIO)
-  rate_set = int(rate * RPCPERF_SET_RATIO)
   nkey = int(ceil(1.0 * slab_mem / (vsize + KSIZE + PELIKAN_ITEM_OVERHEAD)))
   conn_per_thread = connections / threads
 
-  config_str = """\
+  config_str = '''
 [general]
-threads = {threads}
+clients = {threads}
 tcp-nodelay = true
-connections = {connections} # this specifies number of connection per thread
+poolsize = {connections} # this specifies number of connection per thread
 # runtime ~= windows x duration
-windows = 1
-duration = 600
-request-timeout = 200
-connect-timeout = 250
+windows = 2
+interval = 60
+request_ratelimit = {rate}
 
-[[workload]]
-name = "get"
-method = "get"
-rate = {rate_get}
-  [[workload.parameter]]
-  style = "random"
-  size = 32
-  num = {nkey}
-  regenerate = true
+[[keyspace]]
+length = 32
+count = {nkey}
+weight = 1
+commands = [
+    {{action = "get", weight = 9}},
+    {{action = "set", weight = 1}},
+]
+values = [
+    {{length = {vsize}, weight = 1}},
+]'''.format(threads=threads, connections=conn_per_thread, nkey=nkey, rate=rate, vsize=vsize)
 
-[[workload]]
-name = "set"
-method = "set"
-rate = {rate_set}
-  [[workload.parameter]] # key generation parameters
-  style = "random"
-  size = 32
-  num = {nkey}
-  regenerate = true
-  [[workload.parameter]] # value generation parameters
-  style = "random"
-  size = {vsize}
-  regenerate = false
-""".format(threads=threads, connections=conn_per_thread, nkey=nkey, rate_get=rate_get, rate_set=rate_set, vsize=vsize)
   with open('rpcperf.toml', 'w') as the_file:
     the_file.write(config_str)
 
@@ -71,7 +56,9 @@ def generate_runscript(binary, server_ip, instances):
       server_port = PELIKAN_SERVER_PORT + i
       the_file.write('{binary_file} --config {config_file}'.format(binary_file=binary, config_file='rpcperf.toml'))
       the_file.write(' --server {server_ip}:{server_port}'.format(server_ip=server_ip, server_port=server_port))
-      the_file.write(' --waterfall latency-waterfall-{server_port}.png'.format(server_port=server_port))
+      # Currently rpc-perf (prerelease) doesn't support waterfall,
+      # it will be added back soon at which point we can turn this back on
+      # the_file.write(' --waterfall latency-waterfall-{server_port}.png'.format(server_port=server_port))
       the_file.write(' > rpcperf_{server_port}.log'.format(server_port=server_port))
       the_file.write(' 2>&1 &\n')
   os.chmod(fname, 0777)

From ab47258c1ae4fc454671a179048df296ecf80ac5 Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Tue, 15 Jan 2019 14:03:43 -0800
Subject: [PATCH 21/27] further parameterization

---
 scripts/load_testing/client_config.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/scripts/load_testing/client_config.py b/scripts/load_testing/client_config.py
index 09cc63fa7..d7b19f89f 100644
--- a/scripts/load_testing/client_config.py
+++ b/scripts/load_testing/client_config.py
@@ -8,8 +8,8 @@
 RPCPERF_THREADS = 1
 RPCPERF_CONNS = 100
 RPCPERF_RATE = 10000
-RPCPERF_GET_RATIO = 0.9
-RPCPERF_SET_RATIO = 1 - RPCPERF_GET_RATIO
+RPCPERF_GET_WEIGHT = 9
+RPCPERF_SET_WEIGHT = 1
 PELIKAN_SLAB_MEM = 4294967296
 PELIKAN_ITEM_OVERHEAD = 48
 KSIZE = 32
@@ -33,16 +33,17 @@ def generate_config(rate, connections, vsize, slab_mem, threads):
 request_ratelimit = {rate}
 
 [[keyspace]]
-length = 32
+length = {ksize}
 count = {nkey}
 weight = 1
 commands = [
-    {{action = "get", weight = 9}},
-    {{action = "set", weight = 1}},
+    {{action = "get", weight = {get_weight}}},
+    {{action = "set", weight = {get_weight}}},
 ]
 values = [
     {{length = {vsize}, weight = 1}},
-]'''.format(threads=threads, connections=conn_per_thread, nkey=nkey, rate=rate, vsize=vsize)
+]'''.format(threads=threads, connections=conn_per_thread, nkey=nkey, rate=rate,
+    ksize=KSIZE, vsize=vsize, get_weight=PELIKAN_GET_WEIGHT, set_weight=PELIKAN_SET_WEIGHT)
 
   with open('rpcperf.toml', 'w') as the_file:
     the_file.write(config_str)

From 2a0d4e761d22a862d3ef945d394812d884c42336 Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Tue, 15 Jan 2019 14:10:19 -0800
Subject: [PATCH 22/27] misnomer corrected

---
 scripts/load_testing/client_config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/load_testing/client_config.py b/scripts/load_testing/client_config.py
index d7b19f89f..7f54c2d54 100644
--- a/scripts/load_testing/client_config.py
+++ b/scripts/load_testing/client_config.py
@@ -43,7 +43,7 @@ def generate_config(rate, connections, vsize, slab_mem, threads):
 values = [
     {{length = {vsize}, weight = 1}},
 ]'''.format(threads=threads, connections=conn_per_thread, nkey=nkey, rate=rate,
-    ksize=KSIZE, vsize=vsize, get_weight=PELIKAN_GET_WEIGHT, set_weight=PELIKAN_SET_WEIGHT)
+    ksize=KSIZE, vsize=vsize, get_weight=RPCPERF_GET_WEIGHT, set_weight=RPCPERF_SET_WEIGHT)
 
   with open('rpcperf.toml', 'w') as the_file:
     the_file.write(config_str)

From a51bfc64a9c9a1430829c1f69d69db1408949408 Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Tue, 15 Jan 2019 14:11:31 -0800
Subject: [PATCH 23/27] good day for typo

---
 scripts/load_testing/client_config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/load_testing/client_config.py b/scripts/load_testing/client_config.py
index 7f54c2d54..0abc9d13f 100644
--- a/scripts/load_testing/client_config.py
+++ b/scripts/load_testing/client_config.py
@@ -38,7 +38,7 @@ def generate_config(rate, connections, vsize, slab_mem, threads):
 weight = 1
 commands = [
     {{action = "get", weight = {get_weight}}},
-    {{action = "set", weight = {get_weight}}},
+    {{action = "set", weight = {set_weight}}},
 ]
 values = [
     {{length = {vsize}, weight = 1}},

From 3e09291e4ce2aa13aa12c28ebd42d51e139d3133 Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Tue, 15 Jan 2019 14:17:47 -0800
Subject: [PATCH 24/27] more one word change

---
 scripts/load_testing/client_config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/load_testing/client_config.py b/scripts/load_testing/client_config.py
index 0abc9d13f..b4502a0f5 100644
--- a/scripts/load_testing/client_config.py
+++ b/scripts/load_testing/client_config.py
@@ -56,7 +56,7 @@ def generate_runscript(binary, server_ip, instances):
     for i in range(instances):
       server_port = PELIKAN_SERVER_PORT + i
       the_file.write('{binary_file} --config {config_file}'.format(binary_file=binary, config_file='rpcperf.toml'))
-      the_file.write(' --server {server_ip}:{server_port}'.format(server_ip=server_ip, server_port=server_port))
+      the_file.write(' --endpoint {server_ip}:{server_port}'.format(server_ip=server_ip, server_port=server_port))
       # Currently rpc-perf (prerelease) doesn't support waterfall,
       # it will be added back soon at which point we can turn this back on
       # the_file.write(' --waterfall latency-waterfall-{server_port}.png'.format(server_port=server_port))

From f99cce863c62932bc703fd7971690ec55d8acb53 Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Wed, 16 Jan 2019 16:58:39 -0800
Subject: [PATCH 25/27] restoring waterfall now that rpc-perf (prerelease) has
 the feature added back

---
 scripts/load_testing/client_config.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scripts/load_testing/client_config.py b/scripts/load_testing/client_config.py
index b4502a0f5..b094bb20f 100644
--- a/scripts/load_testing/client_config.py
+++ b/scripts/load_testing/client_config.py
@@ -57,9 +57,7 @@ def generate_runscript(binary, server_ip, instances):
       server_port = PELIKAN_SERVER_PORT + i
       the_file.write('{binary_file} --config {config_file}'.format(binary_file=binary, config_file='rpcperf.toml'))
       the_file.write(' --endpoint {server_ip}:{server_port}'.format(server_ip=server_ip, server_port=server_port))
-      # Currently rpc-perf (prerelease) doesn't support waterfall,
-      # it will be added back soon at which point we can turn this back on
-      # the_file.write(' --waterfall latency-waterfall-{server_port}.png'.format(server_port=server_port))
+      the_file.write(' --waterfall latency-waterfall-{server_port}.png'.format(server_port=server_port))
       the_file.write(' > rpcperf_{server_port}.log'.format(server_port=server_port))
       the_file.write(' 2>&1 &\n')
   os.chmod(fname, 0777)

From 8bd14db0700c54329a53800e43656a215c50fb75 Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Tue, 29 Jan 2019 13:17:30 -0800
Subject: [PATCH 26/27] add stats log related config to server

---
 scripts/load_testing/server_config.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/load_testing/server_config.py b/scripts/load_testing/server_config.py
index ffefbbf56..3c96b1901 100644
--- a/scripts/load_testing/server_config.py
+++ b/scripts/load_testing/server_config.py
@@ -65,6 +65,9 @@ def generate_config(instances, vsize, slab_mem):
 slab_mem: {slab_mem}
 slab_size: 1048756
 
+stats_intvl: 10000
+stats_log_file: log/twemcache-{server_port}.stats
+
 time_type: 2
 """.format(admin_port=admin_port, server_port=server_port, vsize=vsize, nkey=nkey, hash_power=hash_power, slab_mem=slab_mem)
     with open(os.path.join('config', config_file),'w') as the_file:

From e0cd439f3f5a520d252832a9b8db949dfb305f63 Mon Sep 17 00:00:00 2001
From: Yao Yue <yao@twitter.com>
Date: Tue, 29 Jan 2019 13:29:31 -0800
Subject: [PATCH 27/27] increase timing wheel size to allow 10 second timeout

---
 scripts/load_testing/server_config.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/load_testing/server_config.py b/scripts/load_testing/server_config.py
index 3c96b1901..c9b48660b 100644
--- a/scripts/load_testing/server_config.py
+++ b/scripts/load_testing/server_config.py
@@ -38,6 +38,8 @@ def generate_config(instances, vsize, slab_mem):
 admin_port: {admin_port}
 server_port: {server_port}
 
+admin_tw_cap: 2000
+
 buf_init_size: 4096
 
 buf_sock_poolsize: 16384