diff --git a/tests/_utils/br_tikv_outage_util b/tests/_utils/br_tikv_outage_util new file mode 100644 index 000000000..7386425af --- /dev/null +++ b/tests/_utils/br_tikv_outage_util @@ -0,0 +1,47 @@ +wait_file_exist() { + timeout=0 + until [ -e "$1" ]; do + timeout=$(( $timeout + 1 )) + if [[ $timeout -gt 100 ]]; then + echo "timeout! maybe BR process is exited!"; exit 1; + fi + sleep 1 + done +} + +single_point_fault() { + type=$1 + victim=$(shuf -i 1-3 -n 1) + echo "Will make failure($type) to store#$victim." + case $type in + outage) + wait_file_exist "$hint_backup_start" + kv_outage -d 30 -i $victim;; + outage-after-request) + wait_file_exist "$hint_get_backup_client" + kv_outage -d 30 -i $victim;; + outage-at-finegrained) + wait_file_exist "$hint_finegrained" + kv_outage --kill -i $victim;; + shutdown) + wait_file_exist "$hint_backup_start" + kv_outage --kill -i $victim;; + scale-out) + wait_file_exist "$hint_backup_start" + kv_outage --kill -i $victim + kv_outage --scale-out -i 4;; + esac +} + +load() { + run_sql "create database if not exists $TEST_NAME" + go-ycsb load mysql -P tests/"$TEST_NAME"/workload -p mysql.host="$TIDB_IP" -p mysql.port="$TIDB_PORT" -p mysql.user=root -p mysql.db="$TEST_NAME" + run_sql 'use '$TEST_NAME'; show tables' +} + +check() { + run_sql 'drop database if exists '$TEST_NAME';' + run_br restore full -s local://"$backup_dir" + count=$(run_sql 'select count(*) from '$TEST_NAME'.usertable;' | tail -n 1 | awk '{print $2}') + [ "$count" -eq 20000 ] +} \ No newline at end of file diff --git a/tests/br_full_ddl/run.sh b/tests/br_full_ddl/run.sh index 1d4ae3e58..4740f7841 100755 --- a/tests/br_full_ddl/run.sh +++ b/tests/br_full_ddl/run.sh @@ -16,7 +16,7 @@ set -eu DB="$TEST_NAME" TABLE="usertable" -DDL_COUNT=10 +DDL_COUNT=5 LOG=/$TEST_DIR/backup.log BACKUP_STAT=/$TEST_DIR/backup_stat RESOTRE_STAT=/$TEST_DIR/restore_stat diff --git a/tests/br_full_ddl/workload b/tests/br_full_ddl/workload index eb81b9903..0a2136bfc 100644 --- a/tests/br_full_ddl/workload +++ b/tests/br_full_ddl/workload @@ -1,7 +1,7 @@ -recordcount=10000 +recordcount=1000 operationcount=0 workload=core -fieldcount=100 +fieldcount=10 readallfields=true @@ -10,4 +10,4 @@ updateproportion=0 scanproportion=0 insertproportion=0 -requestdistribution=uniform \ No newline at end of file +requestdistribution=uniform diff --git a/tests/br_tikv_outage/run.sh b/tests/br_tikv_outage/run.sh index ee9c7be6e..7dd68950e 100644 --- a/tests/br_tikv_outage/run.sh +++ b/tests/br_tikv_outage/run.sh @@ -4,53 +4,7 @@ set -eux . run_services -wait_file_exist() { - timeout=0 - until [ -e "$1" ]; do - timeout=$(( $timeout + 1 )) - if [[ $timeout -gt 100 ]]; then - echo "timeout! maybe BR process is exited!"; exit 1; - fi - sleep 1 - done -} - -single_point_fault() { - type=$1 - victim=$(shuf -i 1-3 -n 1) - echo "Will make failure($type) to store#$victim." - case $type in - outage) - wait_file_exist "$hint_backup_start" - kv_outage -d 30 -i $victim;; - outage-after-request) - wait_file_exist "$hint_get_backup_client" - kv_outage -d 30 -i $victim;; - outage-at-finegrained) - wait_file_exist "$hint_finegrained" - kv_outage --kill -i $victim;; - shutdown) - wait_file_exist "$hint_backup_start" - kv_outage --kill -i $victim;; - scale-out) - wait_file_exist "$hint_backup_start" - kv_outage --kill -i $victim - kv_outage --scale-out -i 4;; - esac -} - -load() { - run_sql "create database if not exists $TEST_NAME" - go-ycsb load mysql -P tests/"$TEST_NAME"/workload -p mysql.host="$TIDB_IP" -p mysql.port="$TIDB_PORT" -p mysql.user=root -p mysql.db="$TEST_NAME" - run_sql 'use '$TEST_NAME'; show tables' -} - -check() { - run_sql 'drop database if exists '$TEST_NAME';' - run_br restore full -s local://"$backup_dir" - count=$(run_sql 'select count(*) from '$TEST_NAME'.usertable;' | tail -n 1 | awk '{print $2}') - [ "$count" -eq 20000 ] -} +. br_tikv_outage_util load @@ -58,18 +12,13 @@ hint_finegrained=$TEST_DIR/hint_finegrained hint_backup_start=$TEST_DIR/hint_backup_start hint_get_backup_client=$TEST_DIR/hint_get_backup_client -# NOTE : cases `outage-at-finegrained shutdown scale-out` should be first to avoid issue -# https://github.com/pingcap/br/issues/1050 -cases=${cases:-'outage-at-finegrained shutdown scale-out outage outage-after-request'} +cases=${cases:-'shutdown scale-out'} for failure in $cases; do rm -f "$hint_finegrained" "$hint_backup_start" "$hint_get_backup_client" export GO_FAILPOINTS="github.com/pingcap/br/pkg/backup/hint-backup-start=1*return(\"$hint_backup_start\");\ github.com/pingcap/br/pkg/backup/hint-fine-grained-backup=1*return(\"$hint_finegrained\");\ github.com/pingcap/br/pkg/conn/hint-get-backup-client=1*return(\"$hint_get_backup_client\")" - if [ "$failure" = outage-at-finegrained ]; then - export GO_FAILPOINTS="$GO_FAILPOINTS;github.com/pingcap/br/pkg/backup/noop-backup=return(true)" - fi backup_dir=${TEST_DIR:?}/"backup{test:${TEST_NAME}|with:${failure}}" rm -rf "${backup_dir:?}" @@ -77,11 +26,10 @@ github.com/pingcap/br/pkg/conn/hint-get-backup-client=1*return(\"$hint_get_backu backup_pid=$! single_point_fault $failure wait $backup_pid - case $failure in - scale-out | shutdown | outage-at-finegrained ) stop_services - start_services ;; - *) ;; - esac + + # both case 'shutdown' and case 'scale-out' need to restart services + stop_services + start_services check diff --git a/tests/br_tikv_outage2/run.sh b/tests/br_tikv_outage2/run.sh new file mode 100644 index 000000000..1e50eafb9 --- /dev/null +++ b/tests/br_tikv_outage2/run.sh @@ -0,0 +1,40 @@ +#! /bin/bash + +set -eux + +. run_services + +. br_tikv_outage_util + +load + +hint_finegrained=$TEST_DIR/hint_finegrained +hint_backup_start=$TEST_DIR/hint_backup_start +hint_get_backup_client=$TEST_DIR/hint_get_backup_client + +cases=${cases:-'outage-at-finegrained outage outage-after-request'} + +for failure in $cases; do + rm -f "$hint_finegrained" "$hint_backup_start" "$hint_get_backup_client" + export GO_FAILPOINTS="github.com/pingcap/br/pkg/backup/hint-backup-start=1*return(\"$hint_backup_start\");\ +github.com/pingcap/br/pkg/backup/hint-fine-grained-backup=1*return(\"$hint_finegrained\");\ +github.com/pingcap/br/pkg/conn/hint-get-backup-client=1*return(\"$hint_get_backup_client\")" + if [ "$failure" = outage-at-finegrained ]; then + export GO_FAILPOINTS="$GO_FAILPOINTS;github.com/pingcap/br/pkg/backup/noop-backup=return(true)" + fi + + backup_dir=${TEST_DIR:?}/"backup{test:${TEST_NAME}|with:${failure}}" + rm -rf "${backup_dir:?}" + run_br backup full -s local://"$backup_dir" & + backup_pid=$! + single_point_fault $failure + wait $backup_pid + case $failure in + scale-out | shutdown | outage-at-finegrained ) stop_services + start_services ;; + *) ;; + esac + + + check +done diff --git a/tests/br_tikv_outage2/workload b/tests/br_tikv_outage2/workload new file mode 100644 index 000000000..de43df83b --- /dev/null +++ b/tests/br_tikv_outage2/workload @@ -0,0 +1,12 @@ +recordcount=20000 +operationcount=0 +workload=core + +readallfields=true + +readproportion=0 +updateproportion=0 +scanproportion=0 +insertproportion=0 + +requestdistribution=uniform