From 8cef447eb5d0e1a4340dd4181ea52af7f203adb9 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Wed, 22 Jun 2022 13:53:02 +0800 Subject: [PATCH 1/7] raw_restore: fix the issue that raw restore rewrite the t prefix keys --- br/tests/br_rawkv/run.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/br/tests/br_rawkv/run.sh b/br/tests/br_rawkv/run.sh index 97450d3e65fc7..33a3d4105b54c 100755 --- a/br/tests/br_rawkv/run.sh +++ b/br/tests/br_rawkv/run.sh @@ -98,8 +98,19 @@ run_test() { --key "$TEST_DIR/certs/br.key" \ --mode put --put-data "311121:31, 31112100:32, 311122:33, 31112200:34, 3111220000:35, 311123:36" + + # put some keys starts with t. https://github.com/pingcap/tidb/issues/35279 + # t_128_r_12 ----> 745f3132385f725f3132 + # t_128_r_13 ----> 745f3132385f725f3133 + bin/rawkv --pd $PD_ADDR \ + --ca "$TEST_DIR/certs/ca.pem" \ + --cert "$TEST_DIR/certs/br.pem" \ + --key "$TEST_DIR/certs/br.key" \ + --mode put --put-data "745f3132385f725f3132, 745f3132385f725f3133" + checksum_ori=$(checksum 31 3130303030303030) checksum_partial=$(checksum 311111 311122) + checksum_t_prefix=$(checksum 745f3132385f725f3132 745f3132385f725f3133) # backup rawkv echo "backup start..." @@ -153,6 +164,21 @@ run_test() { fail_and_exit fi + echo "t prefix restore start..." + run_br --pd $PD_ADDR restore raw -s "local://$BACKUP_DIR" --start "745f3132385f725f3132" --end "745f3132385f725f3133" --format hex --concurrency 4 --crypter.method "aes128-ctr" --crypter.key "0123456789abcdef0123456789abcdef" + bin/rawkv --pd $PD_ADDR \ + --ca "$TEST_DIR/certs/ca.pem" \ + --cert "$TEST_DIR/certs/br.pem" \ + --key "$TEST_DIR/certs/br.key" \ + --mode scan --start-key 745f3132385f725f3132 --end-key 745f3132385f725f3133 + + checksum_new=$(checksum 745f3132385f725f3132 745f3132385f725f3133) + + if [ "$checksum_new" != "$checksum_t_prefix" ];then + echo "checksum failed after restore" + fail_and_exit + fi + export GO_FAILPOINTS="" } From f4e25e8649d4f27ec600ec50e02998f668ef07b8 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Wed, 22 Jun 2022 15:57:42 +0800 Subject: [PATCH 2/7] fix test --- br/tests/br_rawkv/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/br/tests/br_rawkv/run.sh b/br/tests/br_rawkv/run.sh index 33a3d4105b54c..5a0b45c7947f2 100755 --- a/br/tests/br_rawkv/run.sh +++ b/br/tests/br_rawkv/run.sh @@ -106,7 +106,7 @@ run_test() { --ca "$TEST_DIR/certs/ca.pem" \ --cert "$TEST_DIR/certs/br.pem" \ --key "$TEST_DIR/certs/br.key" \ - --mode put --put-data "745f3132385f725f3132, 745f3132385f725f3133" + --mode put --put-data "745f3132385f725f3132:1, 745f3132385f725f3133:2" checksum_ori=$(checksum 31 3130303030303030) checksum_partial=$(checksum 311111 311122) From f9c92866bf4bd533a1651c8981b0c15f503f6c9e Mon Sep 17 00:00:00 2001 From: 3pointer Date: Wed, 22 Jun 2022 19:10:29 +0800 Subject: [PATCH 3/7] fix test --- br/tests/br_rawkv/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/br/tests/br_rawkv/run.sh b/br/tests/br_rawkv/run.sh index 5a0b45c7947f2..a5ac2aa5e033a 100755 --- a/br/tests/br_rawkv/run.sh +++ b/br/tests/br_rawkv/run.sh @@ -106,7 +106,7 @@ run_test() { --ca "$TEST_DIR/certs/ca.pem" \ --cert "$TEST_DIR/certs/br.pem" \ --key "$TEST_DIR/certs/br.key" \ - --mode put --put-data "745f3132385f725f3132:1, 745f3132385f725f3133:2" + --mode put --put-data "745f3132385f725f3132:31, 745f3132385f725f3133:32" checksum_ori=$(checksum 31 3130303030303030) checksum_partial=$(checksum 311111 311122) From 4bbd9dabdcde75c2df8e06584b2b1a4e262c60e2 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Wed, 22 Jun 2022 19:44:40 +0800 Subject: [PATCH 4/7] fix --- br/pkg/task/restore_raw.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/br/pkg/task/restore_raw.go b/br/pkg/task/restore_raw.go index d8fcb46475809..452cccfad8c42 100644 --- a/br/pkg/task/restore_raw.go +++ b/br/pkg/task/restore_raw.go @@ -146,8 +146,7 @@ func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreR !cfg.LogProgress) // RawKV restore does not need to rewrite keys. - rewrite := &restore.RewriteRules{} - err = restore.SplitRanges(ctx, client, ranges, rewrite, updateCh, true) + err = restore.SplitRanges(ctx, client, ranges, nil, updateCh, true) if err != nil { return errors.Trace(err) } From 76536b660e7bef71b314bd0f81dc391b0b49cad6 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Wed, 22 Jun 2022 20:29:02 +0800 Subject: [PATCH 5/7] fix --- br/tests/br_rawkv/run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/br/tests/br_rawkv/run.sh b/br/tests/br_rawkv/run.sh index a5ac2aa5e033a..cbe31a87ccbc9 100755 --- a/br/tests/br_rawkv/run.sh +++ b/br/tests/br_rawkv/run.sh @@ -110,7 +110,7 @@ run_test() { checksum_ori=$(checksum 31 3130303030303030) checksum_partial=$(checksum 311111 311122) - checksum_t_prefix=$(checksum 745f3132385f725f3132 745f3132385f725f3133) + checksum_t_prefix=$(checksum 745f3132385f725f3131 745f3132385f725f3134) # backup rawkv echo "backup start..." @@ -165,14 +165,14 @@ run_test() { fi echo "t prefix restore start..." - run_br --pd $PD_ADDR restore raw -s "local://$BACKUP_DIR" --start "745f3132385f725f3132" --end "745f3132385f725f3133" --format hex --concurrency 4 --crypter.method "aes128-ctr" --crypter.key "0123456789abcdef0123456789abcdef" + run_br --pd $PD_ADDR restore raw -s "local://$BACKUP_DIR" --start "745f3132385f725f3131" --end "745f3132385f725f3134" --format hex --concurrency 4 --crypter.method "aes128-ctr" --crypter.key "0123456789abcdef0123456789abcdef" bin/rawkv --pd $PD_ADDR \ --ca "$TEST_DIR/certs/ca.pem" \ --cert "$TEST_DIR/certs/br.pem" \ --key "$TEST_DIR/certs/br.key" \ - --mode scan --start-key 745f3132385f725f3132 --end-key 745f3132385f725f3133 + --mode scan --start-key 745f3132385f725f3131 --end-key 745f3132385f725f3134 - checksum_new=$(checksum 745f3132385f725f3132 745f3132385f725f3133) + checksum_new=$(checksum 745f3132385f725f3131 745f3132385f725f3133) if [ "$checksum_new" != "$checksum_t_prefix" ];then echo "checksum failed after restore" From ec8e5ddd5d9b94d76f7166657d1a463174aed353 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Wed, 22 Jun 2022 21:04:11 +0800 Subject: [PATCH 6/7] fix --- br/tests/br_rawkv/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/br/tests/br_rawkv/run.sh b/br/tests/br_rawkv/run.sh index cbe31a87ccbc9..fe01f35e9d85d 100755 --- a/br/tests/br_rawkv/run.sh +++ b/br/tests/br_rawkv/run.sh @@ -114,7 +114,7 @@ run_test() { # backup rawkv echo "backup start..." - run_br --pd $PD_ADDR backup raw -s "local://$BACKUP_DIR" --start 31 --end 3130303030303030 --format hex --concurrency 4 --crypter.method "aes128-ctr" --crypter.key "0123456789abcdef0123456789abcdef" + run_br --pd $PD_ADDR backup raw -s "local://$BACKUP_DIR" --start 31 --end 745f3132385f725f3134 --format hex --concurrency 4 --crypter.method "aes128-ctr" --crypter.key "0123456789abcdef0123456789abcdef" # delete data in range[start-key, end-key) clean 31 3130303030303030 From 26a09ac2cf1ea2470906d9cd1f0be5c093babea5 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Thu, 23 Jun 2022 10:27:53 +0800 Subject: [PATCH 7/7] fix test --- br/tests/br_rawkv/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/br/tests/br_rawkv/run.sh b/br/tests/br_rawkv/run.sh index fe01f35e9d85d..b32cca0f8e41f 100755 --- a/br/tests/br_rawkv/run.sh +++ b/br/tests/br_rawkv/run.sh @@ -172,7 +172,7 @@ run_test() { --key "$TEST_DIR/certs/br.key" \ --mode scan --start-key 745f3132385f725f3131 --end-key 745f3132385f725f3134 - checksum_new=$(checksum 745f3132385f725f3131 745f3132385f725f3133) + checksum_new=$(checksum 745f3132385f725f3131 745f3132385f725f3134) if [ "$checksum_new" != "$checksum_t_prefix" ];then echo "checksum failed after restore"