Optimize integration test. (#1274) (#1310)

tests/_utils/br_tikv_outage_util

@@ -0,0 +1,47 @@
+wait_file_exist() {
+    timeout=0
+    until [ -e "$1" ]; do
+        timeout=$(( $timeout + 1 ))
+        if [[ $timeout -gt 100 ]]; then
+            echo "timeout! maybe BR process is exited!"; exit 1;
+        fi
+        sleep 1
+    done
+}
+
+single_point_fault() {
+    type=$1
+    victim=$(shuf -i 1-3 -n 1)
+    echo "Will make failure($type) to store#$victim."
+    case $type in
+        outage)
+            wait_file_exist "$hint_backup_start"
+            kv_outage -d 30 -i $victim;;
+        outage-after-request)
+            wait_file_exist "$hint_get_backup_client"
+            kv_outage -d 30 -i $victim;;
+        outage-at-finegrained)
+            wait_file_exist "$hint_finegrained"
+            kv_outage --kill -i $victim;;
+        shutdown)
+            wait_file_exist "$hint_backup_start"
+            kv_outage --kill -i $victim;;
+        scale-out)
+            wait_file_exist "$hint_backup_start"
+            kv_outage --kill -i $victim
+            kv_outage --scale-out -i 4;;
+    esac
+}
+
+load() {
+    run_sql "create database if not exists $TEST_NAME"
+    go-ycsb load mysql -P tests/"$TEST_NAME"/workload -p mysql.host="$TIDB_IP" -p mysql.port="$TIDB_PORT" -p mysql.user=root -p mysql.db="$TEST_NAME"
+    run_sql 'use '$TEST_NAME'; show tables'
+}
+
+check() {
+    run_sql 'drop database if exists '$TEST_NAME';'
+    run_br restore full -s local://"$backup_dir" 
+    count=$(run_sql 'select count(*) from '$TEST_NAME'.usertable;' | tail -n 1 | awk '{print $2}')
+    [ "$count" -eq 20000 ]
+}

tests/br_full_ddl/run.sh

@@ -16,7 +16,7 @@
 set -eu
 DB="$TEST_NAME"
 TABLE="usertable"
-DDL_COUNT=10
+DDL_COUNT=5
 LOG=/$TEST_DIR/backup.log
 BACKUP_STAT=/$TEST_DIR/backup_stat
 RESOTRE_STAT=/$TEST_DIR/restore_stat

tests/br_full_ddl/workload

@@ -1,7 +1,7 @@
-recordcount=10000
+recordcount=1000
 operationcount=0
 workload=core
-fieldcount=100
+fieldcount=10
 
 readallfields=true
 
@@ -10,4 +10,4 @@ updateproportion=0
 scanproportion=0
 insertproportion=0
 
-requestdistribution=uniform
+requestdistribution=uniform

tests/br_tikv_outage/run.sh

@@ -4,84 +4,32 @@ set -eux
 
 . run_services
 
-wait_file_exist() {
-    timeout=0
-    until [ -e "$1" ]; do
-        timeout=$(( $timeout + 1 ))
-        if [[ $timeout -gt 100 ]]; then
-            echo "timeout! maybe BR process is exited!"; exit 1;
-        fi
-        sleep 1
-    done
-}
-
-single_point_fault() {
-    type=$1
-    victim=$(shuf -i 1-3 -n 1)
-    echo "Will make failure($type) to store#$victim."
-    case $type in
-        outage)
-            wait_file_exist "$hint_backup_start"
-            kv_outage -d 30 -i $victim;;
-        outage-after-request)
-            wait_file_exist "$hint_get_backup_client"
-            kv_outage -d 30 -i $victim;;
-        outage-at-finegrained)
-            wait_file_exist "$hint_finegrained"
-            kv_outage --kill -i $victim;;
-        shutdown)
-            wait_file_exist "$hint_backup_start"
-            kv_outage --kill -i $victim;;
-        scale-out)
-            wait_file_exist "$hint_backup_start"
-            kv_outage --kill -i $victim
-            kv_outage --scale-out -i 4;;
-    esac
-}
-
-load() {
-    run_sql "create database if not exists $TEST_NAME"
-    go-ycsb load mysql -P tests/"$TEST_NAME"/workload -p mysql.host="$TIDB_IP" -p mysql.port="$TIDB_PORT" -p mysql.user=root -p mysql.db="$TEST_NAME"
-    run_sql 'use '$TEST_NAME'; show tables'
-}
-
-check() {
-    run_sql 'drop database if exists '$TEST_NAME';'
-    run_br restore full -s local://"$backup_dir" 
-    count=$(run_sql 'select count(*) from '$TEST_NAME'.usertable;' | tail -n 1 | awk '{print $2}')
-    [ "$count" -eq 20000 ]
-}
+. br_tikv_outage_util
 
 load
 
 hint_finegrained=$TEST_DIR/hint_finegrained
 hint_backup_start=$TEST_DIR/hint_backup_start
 hint_get_backup_client=$TEST_DIR/hint_get_backup_client
 
-# NOTE : cases `outage-at-finegrained shutdown scale-out` should be first to avoid issue
-#       https://github.com/pingcap/br/issues/1050
-cases=${cases:-'outage-at-finegrained shutdown scale-out outage outage-after-request'}
+cases=${cases:-'shutdown scale-out'}
 
 for failure in $cases; do
     rm -f "$hint_finegrained" "$hint_backup_start" "$hint_get_backup_client"
     export GO_FAILPOINTS="github.com/pingcap/br/pkg/backup/hint-backup-start=1*return(\"$hint_backup_start\");\
 github.com/pingcap/br/pkg/backup/hint-fine-grained-backup=1*return(\"$hint_finegrained\");\
 github.com/pingcap/br/pkg/conn/hint-get-backup-client=1*return(\"$hint_get_backup_client\")"
-    if [ "$failure" = outage-at-finegrained ]; then
-        export GO_FAILPOINTS="$GO_FAILPOINTS;github.com/pingcap/br/pkg/backup/noop-backup=return(true)"
-    fi
 
     backup_dir=${TEST_DIR:?}/"backup{test:${TEST_NAME}|with:${failure}}"
     rm -rf "${backup_dir:?}"
     run_br backup full -s local://"$backup_dir" &
     backup_pid=$!
     single_point_fault $failure
     wait $backup_pid
-    case $failure in
-    scale-out | shutdown | outage-at-finegrained ) stop_services
-        start_services ;;
-    *) ;;
-    esac
+
+    # both case 'shutdown' and case 'scale-out' need to restart services
+    stop_services
+    start_services
 
 
     check

tests/br_tikv_outage2/run.sh

@@ -0,0 +1,40 @@
+#! /bin/bash
+
+set -eux
+
+. run_services
+
+. br_tikv_outage_util
+
+load
+
+hint_finegrained=$TEST_DIR/hint_finegrained
+hint_backup_start=$TEST_DIR/hint_backup_start
+hint_get_backup_client=$TEST_DIR/hint_get_backup_client
+
+cases=${cases:-'outage-at-finegrained outage outage-after-request'}
+
+for failure in $cases; do
+    rm -f "$hint_finegrained" "$hint_backup_start" "$hint_get_backup_client"
+    export GO_FAILPOINTS="github.com/pingcap/br/pkg/backup/hint-backup-start=1*return(\"$hint_backup_start\");\
+github.com/pingcap/br/pkg/backup/hint-fine-grained-backup=1*return(\"$hint_finegrained\");\
+github.com/pingcap/br/pkg/conn/hint-get-backup-client=1*return(\"$hint_get_backup_client\")"
+    if [ "$failure" = outage-at-finegrained ]; then
+        export GO_FAILPOINTS="$GO_FAILPOINTS;github.com/pingcap/br/pkg/backup/noop-backup=return(true)"
+    fi
+
+    backup_dir=${TEST_DIR:?}/"backup{test:${TEST_NAME}|with:${failure}}"
+    rm -rf "${backup_dir:?}"
+    run_br backup full -s local://"$backup_dir" &
+    backup_pid=$!
+    single_point_fault $failure
+    wait $backup_pid
+    case $failure in
+    scale-out | shutdown | outage-at-finegrained ) stop_services
+        start_services ;;
+    *) ;;
+    esac
+
+
+    check
+done

tests/br_tikv_outage2/workload

@@ -0,0 +1,12 @@
+recordcount=20000
+operationcount=0
+workload=core
+
+readallfields=true
+
+readproportion=0
+updateproportion=0
+scanproportion=0
+insertproportion=0
+
+requestdistribution=uniform