diff --git a/perfscale_regression_ci/scripts/custom_workload_env.sh b/perfscale_regression_ci/scripts/custom_workload_env.sh index 6a98a2830ca..b002b496f36 100644 --- a/perfscale_regression_ci/scripts/custom_workload_env.sh +++ b/perfscale_regression_ci/scripts/custom_workload_env.sh @@ -1,6 +1,3 @@ -# Where to clone e2e-benchmarking from -export E2E_BENCHMARKING_REPOSITORY=${E2E_BENCHMARKING_REPOSITORY:-"https://github.com/cloud-bulldozer/e2e-benchmarking"} -export E2E_BENCHMARKING_BRANCH=${E2E_BENCHMARKING_BRANCH:-"master"} # Common ENVs of custom workload for kube-burner export WORKLOAD=${WORKLOAD:-"custom"} export INDEXING=${INDEXING:-"false"} diff --git a/perfscale_regression_ci/scripts/scalability/router-scale/router-scale.sh b/perfscale_regression_ci/scripts/scalability/router-scale/router-scale.sh new file mode 100755 index 00000000000..d46cc2d0e63 --- /dev/null +++ b/perfscale_regression_ci/scripts/scalability/router-scale/router-scale.sh @@ -0,0 +1,51 @@ +#/!/bin/bash +################################################ +## Auth=qili@redhat.com +## Desription: Script for router scale test +## Polarion test case: OCP-43281 +## https://polarion.engineering.redhat.com/polarion/#/project/OSE/workitem?id=OCP-43281 +## Cluster config: 3 master (m5.2xlarge), 9 workers(m5.xlarge), 3 infras(c5.4xlarge), no not move component to infra nodes before the test +## ingress-perf config: standard-3replicas.yml, standard-4replicas.yml +## optional PARAMETERS: number of JOB_ITERATION +################################################ +set -o errexit + +source ../../common.sh +source ../../../utils/run_workload.sh + +echo "[INFO] Patch threadCount as 4" +oc -n openshift-ingress-operator patch ingresscontroller/default --type=merge -p '{"spec":{"tuningOptions": {"threadCount": 4}}}' +oc -n openshift-ingress get deploy router-default -o yaml | grep " ROUTER_THREADS" -A 1 + +echo "[INFO] run ingress-perf with thread=4, replica=2" +run_ingress_perf +#run ingress-perf with thread=4, replica=3 +export CONFIG=../../../standard-3replicas.yml +run_ingress_perf + +echo "[INFO] scale infra replicas to 4" +machineset=$(oc get machinesets -n openshift-machine-api --no-headers | head -n 1 | awk {'print $1'}) +oc scale machineset --replicas=2 ${machineset} -n openshift-machine-api +oc get machinesets -n openshift-machine-api + +echo "[INFO] run ingress-perf with thread=4, replica=4" +export CONFIG=../../../standard-4replicas.yml +run_ingress_perf + +echo "[INFO] Patch threadCount as 8" +oc -n openshift-ingress-operator patch ingresscontroller/default --type=merge -p '{"spec":{"tuningOptions": {"threadCount": 8}}}' +oc -n openshift-ingress get deploy router-default -o yaml | grep " ROUTER_THREADS" -A 1 + +echo "[INFO] run ingress-perf with thread=8, replica=4" +export CONFIG=../../../standard-4replicas.yml +run_ingress_perf + +echo "[INFO] run ingress-perf with thread=8, replica=3" +export CONFIG=../../../standard-3replicas.yml +run_ingress_perf + +echo "[INFO] run ingress-perf with thread=8, replica=2" +unset CONFIG +run_ingress_perf + +echo "[INFO] Test is finished. Pleaset check results in the grafana and dittybopper dashboards." \ No newline at end of file diff --git a/perfscale_regression_ci/scripts/scalability/router-scale/standard-3replicas.yml b/perfscale_regression_ci/scripts/scalability/router-scale/standard-3replicas.yml new file mode 100755 index 00000000000..12c911d6a75 --- /dev/null +++ b/perfscale_regression_ci/scripts/scalability/router-scale/standard-3replicas.yml @@ -0,0 +1,82 @@ +# vi: expandtab shiftwidth=2 softtabstop=2 + +# First scenario is configured as warmup and it will also tune the default ingress-controller to assign the router pods to the infra nodes +- termination: http + connections: 200 + samples: 5 + duration: 3m + path: /1024.html + concurrency: 18 + tool: wrk + serverReplicas: 45 + tuningPatch: '{"spec":{"nodePlacement": {"nodeSelector": {"matchLabels": {"node-role.kubernetes.io/infra": ""}}}, "replicas": 3}}' + delay: 10s + requestTimeout: 10s + warmup: true + +- termination: http + connections: 200 + samples: 2 + duration: 3m + path: /1024.html + concurrency: 18 + tool: wrk + serverReplicas: 45 + requestTimeout: 10s + delay: 10s + +- termination: edge + connections: 200 + samples: 2 + duration: 3m + path: /1024.html + concurrency: 18 + tool: wrk + serverReplicas: 45 + requestTimeout: 10s + delay: 10s + +- termination: reencrypt + connections: 200 + samples: 2 + duration: 3m + path: /1024.html + concurrency: 18 + tool: wrk + serverReplicas: 45 + requestTimeout: 10s + delay: 10s + +- termination: passthrough + connections: 200 + samples: 2 + duration: 3m + path: /1024.html + concurrency: 18 + tool: wrk + serverReplicas: 45 + requestTimeout: 10s + delay: 10s + +- termination: reencrypt + connections: 200 + samples: 2 + duration: 3m + path: /1024.html + concurrency: 18 + tool: hloader + serverReplicas: 45 + requestTimeout: 10s + tuningPatch: '{"metadata":{"annotations": {"ingress.operator.openshift.io/default-enable-http2": "true"}}}' + http2: true + +- termination: passthrough + connections: 200 + samples: 2 + duration: 3m + path: /1024.html + concurrency: 18 + tool: hloader + serverReplicas: 45 + requestTimeout: 10s + http2: true \ No newline at end of file diff --git a/perfscale_regression_ci/scripts/scalability/router-scale/standard-4replicas.yml b/perfscale_regression_ci/scripts/scalability/router-scale/standard-4replicas.yml new file mode 100755 index 00000000000..5a87b18dbbd --- /dev/null +++ b/perfscale_regression_ci/scripts/scalability/router-scale/standard-4replicas.yml @@ -0,0 +1,82 @@ +# vi: expandtab shiftwidth=2 softtabstop=2 + +# First scenario is configured as warmup and it will also tune the default ingress-controller to assign the router pods to the infra nodes +- termination: http + connections: 200 + samples: 5 + duration: 3m + path: /1024.html + concurrency: 18 + tool: wrk + serverReplicas: 45 + tuningPatch: '{"spec":{"nodePlacement": {"nodeSelector": {"matchLabels": {"node-role.kubernetes.io/infra": ""}}}, "replicas": 4}}' + delay: 10s + requestTimeout: 10s + warmup: true + +- termination: http + connections: 200 + samples: 2 + duration: 3m + path: /1024.html + concurrency: 18 + tool: wrk + serverReplicas: 45 + requestTimeout: 10s + delay: 10s + +- termination: edge + connections: 200 + samples: 2 + duration: 3m + path: /1024.html + concurrency: 18 + tool: wrk + serverReplicas: 45 + requestTimeout: 10s + delay: 10s + +- termination: reencrypt + connections: 200 + samples: 2 + duration: 3m + path: /1024.html + concurrency: 18 + tool: wrk + serverReplicas: 45 + requestTimeout: 10s + delay: 10s + +- termination: passthrough + connections: 200 + samples: 2 + duration: 3m + path: /1024.html + concurrency: 18 + tool: wrk + serverReplicas: 45 + requestTimeout: 10s + delay: 10s + +- termination: reencrypt + connections: 200 + samples: 2 + duration: 3m + path: /1024.html + concurrency: 18 + tool: hloader + serverReplicas: 45 + requestTimeout: 10s + tuningPatch: '{"metadata":{"annotations": {"ingress.operator.openshift.io/default-enable-http2": "true"}}}' + http2: true + +- termination: passthrough + connections: 200 + samples: 2 + duration: 3m + path: /1024.html + concurrency: 18 + tool: hloader + serverReplicas: 45 + requestTimeout: 10s + http2: true \ No newline at end of file diff --git a/perfscale_regression_ci/utils/run_workload.sh b/perfscale_regression_ci/utils/run_workload.sh index 8e91592ae45..15725ef39e1 100644 --- a/perfscale_regression_ci/utils/run_workload.sh +++ b/perfscale_regression_ci/utils/run_workload.sh @@ -4,7 +4,12 @@ set -ex setup(){ rm -rf e2e-benchmarking - git clone --single-branch --branch ${E2E_BENCHMARKING_BRANCH} ${E2E_BENCHMARKING_REPOSITORY} + # Clone the e2e repo + REPO_URL="https://github.com/cloud-bulldozer/e2e-benchmarking"; + LATEST_TAG=$(curl -s "https://api.github.com/repos/cloud-bulldozer/e2e-benchmarking/releases/latest" | jq -r '.tag_name'); + E2E_VERSION="v2.2.5" + TAG_OPTION="--branch $(if [ "$E2E_VERSION" == "default" ]; then echo "$LATEST_TAG"; else echo "$E2E_VERSION"; fi)"; + git clone $REPO_URL $TAG_OPTION --depth 1 } cleanup(){ @@ -21,4 +26,19 @@ run_workload(){ cleanup } -set +ex +run_ingress_perf(){ + if [[ ! -d e2e-benchmarking/workloads/ingress-perf ]]; then + setup + fi + pushd e2e-benchmarking/workloads/ingress-perf + export ES_USERNAME=${ES_USERNAME} + export ES_PASSWORD=${ES_PASSWORD} + export ES_SERVER="https://$ES_USERNAME:$ES_PASSWORD@search-ocp-qe-perf-scale-test-elk-hcm7wtsqpxy7xogbu72bor4uve.us-east-1.es.amazonaws.com" + export ES_INDEX="ingress-performance" + export CONFIG=${CONFIG:-"config/standard.yml"} + echo "[INFO] Will run ingress-perf with config $CONFIG" + ./run.sh |& tee "/tmp/ingress-perf-$(date +%Y%m%d%H%M%S).out" + popd +} + +set +ex \ No newline at end of file