diff --git a/br/pkg/lightning/backend/tidb/tidb_test.go b/br/pkg/lightning/backend/tidb/tidb_test.go index b7c13445d39a1..8f2f9a48af6e4 100644 --- a/br/pkg/lightning/backend/tidb/tidb_test.go +++ b/br/pkg/lightning/backend/tidb/tidb_test.go @@ -690,7 +690,7 @@ func TestWriteRowsRecordOneError(t *testing.T) { ExpectExec("\\QINSERT INTO `foo`.`bar`(`a`) VALUES(2)\\E"). WillReturnError(dupErr) s.mockDB. - ExpectExec("INSERT INTO `tidb_lightning_errors`\\.duplicate_records.*"). + ExpectExec("INSERT INTO `tidb_lightning_errors`\\.conflict_records.*"). WithArgs(sqlmock.AnyArg(), "`foo`.`bar`", "8.csv", int64(0), dupErr.Error(), 0, "(2)"). WillReturnResult(driver.ResultNoRows) diff --git a/br/pkg/lightning/errormanager/errormanager.go b/br/pkg/lightning/errormanager/errormanager.go index 6466d44b6463f..ca5e37c154f7d 100644 --- a/br/pkg/lightning/errormanager/errormanager.go +++ b/br/pkg/lightning/errormanager/errormanager.go @@ -46,7 +46,7 @@ const ( // ConflictErrorTableName is the table name for duplicate detection. ConflictErrorTableName = "conflict_error_v1" // DupRecordTable is the table name to record duplicate data that displayed to user. - DupRecordTable = "duplicate_records" + DupRecordTable = "conflict_records" createSyntaxErrorTable = ` CREATE TABLE IF NOT EXISTS %s.` + syntaxErrorTableName + ` ( diff --git a/br/pkg/lightning/errormanager/errormanager_test.go b/br/pkg/lightning/errormanager/errormanager_test.go index 7a84a90da87dd..2e5c913b71f40 100644 --- a/br/pkg/lightning/errormanager/errormanager_test.go +++ b/br/pkg/lightning/errormanager/errormanager_test.go @@ -73,7 +73,7 @@ func TestInit(t *testing.T) { WillReturnResult(sqlmock.NewResult(6, 1)) mock.ExpectExec("CREATE TABLE IF NOT EXISTS `lightning_errors`\\.conflict_error_v1.*"). WillReturnResult(sqlmock.NewResult(7, 1)) - mock.ExpectExec("CREATE TABLE IF NOT EXISTS `lightning_errors`\\.duplicate_records.*"). + mock.ExpectExec("CREATE TABLE IF NOT EXISTS `lightning_errors`\\.conflict_records.*"). WillReturnResult(sqlmock.NewResult(7, 1)) err = em.Init(ctx) require.NoError(t, err) @@ -291,13 +291,13 @@ func TestErrorMgrErrorOutput(t *testing.T) { output = em.Output() expected = "\n" + "Import Data Error Summary: \n" + - "+---+---------------------+-------------+----------------------------------+\n" + - "| # | ERROR TYPE | ERROR COUNT | ERROR DATA TABLE |\n" + - "+---+---------------------+-------------+----------------------------------+\n" + - "|\x1b[31m 1 \x1b[0m|\x1b[31m Data Type \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`type_error_v1` \x1b[0m|\n" + - "|\x1b[31m 2 \x1b[0m|\x1b[31m Data Syntax \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`syntax_error_v1` \x1b[0m|\n" + - "|\x1b[31m 3 \x1b[0m|\x1b[31m Charset Error \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m \x1b[0m|\n" + - "|\x1b[31m 4 \x1b[0m|\x1b[31m Unique Key Conflict \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`duplicate_records` \x1b[0m|\n" + - "+---+---------------------+-------------+----------------------------------+\n" + "+---+---------------------+-------------+---------------------------------+\n" + + "| # | ERROR TYPE | ERROR COUNT | ERROR DATA TABLE |\n" + + "+---+---------------------+-------------+---------------------------------+\n" + + "|\x1b[31m 1 \x1b[0m|\x1b[31m Data Type \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`type_error_v1` \x1b[0m|\n" + + "|\x1b[31m 2 \x1b[0m|\x1b[31m Data Syntax \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`syntax_error_v1` \x1b[0m|\n" + + "|\x1b[31m 3 \x1b[0m|\x1b[31m Charset Error \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m \x1b[0m|\n" + + "|\x1b[31m 4 \x1b[0m|\x1b[31m Unique Key Conflict \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`conflict_records` \x1b[0m|\n" + + "+---+---------------------+-------------+---------------------------------+\n" require.Equal(t, expected, output) } diff --git a/br/tests/lightning_config_max_error/run.sh b/br/tests/lightning_config_max_error/run.sh index 6b5293d29e7d7..543971140c015 100755 --- a/br/tests/lightning_config_max_error/run.sh +++ b/br/tests/lightning_config_max_error/run.sh @@ -80,20 +80,20 @@ check_contains "COUNT(*): ${duplicated_row_count}" run_sql 'SELECT COUNT(*) FROM mytest.testtbl' check_contains "COUNT(*): ${uniq_row_count}" -# Check tidb backend record duplicate entry in duplicate_records table -run_sql 'DROP TABLE IF EXISTS lightning_task_info.duplicate_records' +# Check tidb backend record duplicate entry in conflict_records table +run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_records' run_lightning --backend tidb --config "${mydir}/tidb.toml" -run_sql 'SELECT COUNT(*) FROM lightning_task_info.duplicate_records' +run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_records' check_contains "COUNT(*): 10" -run_sql 'SELECT * FROM lightning_task_info.duplicate_records WHERE offset = 149' +run_sql 'SELECT * FROM lightning_task_info.conflict_records WHERE offset = 149' check_contains "error: Error 1062 (23000): Duplicate entry '5' for key 'testtbl.PRIMARY'" check_contains "row_data: ('5','bbb05')" -# Check max-error-record can limit the size of duplicate_records table +# Check max-error-record can limit the size of conflict_records table run_sql 'DROP DATABASE IF EXISTS lightning_task_info' run_sql 'DROP DATABASE IF EXISTS mytest' -run_lightning --backend tidb --config "${mydir}/tidb-limit-record.toml" 2>&1 | grep "\`lightning_task_info\`.\`duplicate_records\`" | grep -q "5" -run_sql 'SELECT COUNT(*) FROM lightning_task_info.duplicate_records' +run_lightning --backend tidb --config "${mydir}/tidb-limit-record.toml" 2>&1 | grep "\`lightning_task_info\`.\`conflict_records\`" | grep -q "5" +run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_records' check_contains "COUNT(*): 1" # Check conflict.threshold @@ -109,8 +109,8 @@ run_sql 'DROP DATABASE IF EXISTS mytest' rm "${TEST_DIR}/lightning.log" run_lightning --backend tidb --config "${mydir}/tidb-error.toml" 2>&1 | grep -q "Error 1062 (23000): Duplicate entry '1' for key 'testtbl.PRIMARY'" check_contains "Error 1062 (23000): Duplicate entry '1' for key 'testtbl.PRIMARY'" "${TEST_DIR}/lightning.log" -run_sql 'SELECT COUNT(*) FROM lightning_task_info.duplicate_records' +run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_records' check_contains "COUNT(*): 1" -run_sql 'SELECT * FROM lightning_task_info.duplicate_records' +run_sql 'SELECT * FROM lightning_task_info.conflict_records' check_contains "error: Error 1062 (23000): Duplicate entry '1' for key 'testtbl.PRIMARY'" check_contains "row_data: ('1','bbb01')" diff --git a/br/tests/lightning_duplicate_detection_new/run.sh b/br/tests/lightning_duplicate_detection_new/run.sh index 955fee41da21d..247dfd871c4c8 100755 --- a/br/tests/lightning_duplicate_detection_new/run.sh +++ b/br/tests/lightning_duplicate_detection_new/run.sh @@ -40,14 +40,14 @@ if [ "$expected_rows" != "$actual_rows" ] || [ "$expected_pks" != "$actual_pks" echo "local backend replace strategy result is not equal to tidb backend" exit 1 fi -run_sql "SELECT count(*) FROM lightning_task_info.duplicate_records" +run_sql "SELECT count(*) FROM lightning_task_info.conflict_records" check_contains "count(*): 227" -run_sql "SELECT count(*) FROM lightning_task_info.duplicate_records WHERE error = ''" +run_sql "SELECT count(*) FROM lightning_task_info.conflict_records WHERE error = ''" check_contains "count(*): 0" -run_sql "SELECT * FROM lightning_task_info.duplicate_records WHERE row_id = 12" +run_sql "SELECT * FROM lightning_task_info.conflict_records WHERE row_id = 12" check_contains "(171,'yRxZE',9201592769833450947,'xs3d',5,4,283270321)" check_contains "[kv:1062]Duplicate entry '171' for key 'dup_detect.PRIMARY'" -run_sql "SELECT * FROM lightning_task_info.duplicate_records WHERE row_id = 1" +run_sql "SELECT * FROM lightning_task_info.conflict_records WHERE row_id = 1" check_contains "(87,'nEoKu',7836621565948506759,'y6',48,0,177543185)" check_contains "[kv:1062]Duplicate entry '0-177543185' for key 'dup_detect.uniq_col6_col7'" @@ -65,18 +65,18 @@ if [ "$expected_rows" != "$actual_rows" ] || [ "$expected_pks" != "$actual_pks" echo "local backend ignore strategy result is not equal to tidb backend" exit 1 fi -run_sql "SELECT count(*) FROM lightning_task_info.duplicate_records" +run_sql "SELECT count(*) FROM lightning_task_info.conflict_records" check_contains "count(*): 228" # 3. Test error strategy. cleanup run_lightning --backend local --config "$CUR/local-error.toml" --log-file "$LOG_FILE" 2>&1 | grep -q "duplicate key in table \`test\`.\`dup_detect\` caused by index .*, but because checkpoint is off we can't have more details" grep -q "duplicate key in table \`test\`.\`dup_detect\` caused by index .*, but because checkpoint is off we can't have more details" "$LOG_FILE" -run_sql "SELECT * FROM lightning_task_info.duplicate_records" +run_sql "SELECT * FROM lightning_task_info.conflict_records" check_contains "error: duplicate key in table \`test\`.\`dup_detect\`" run_lightning --backend local --config "$CUR/local-error.toml" --log-file "$LOG_FILE" --enable-checkpoint=1 2>&1 | grep -q "duplicate entry for key 'uniq_col6_col7', a pair of conflicting rows are (row 1 counting from offset 0 in file test.dup_detect.1.sql, row 101 counting from offset 0 in file test.dup_detect.4.sql)" grep -q "duplicate entry for key 'uniq_col6_col7', a pair of conflicting rows are (row 1 counting from offset 0 in file test.dup_detect.1.sql, row 101 counting from offset 0 in file test.dup_detect.4.sql)" "$LOG_FILE" -run_sql "SELECT * FROM lightning_task_info.duplicate_records" +run_sql "SELECT * FROM lightning_task_info.conflict_records" check_contains "error: duplicate entry for key 'uniq_col6_col7', a pair of conflicting rows are" check_contains "restore table \`test\`.\`dup_detect\` failed: duplicate entry for key 'uniq_col6_col7', a pair of conflicting rows are (row 1 counting from offset 0 in file test.dup_detect.1.sql, row 101 counting from offset 0 in file test.dup_detect.4.sql)" "$LOG_FILE" run_lightning_ctl --enable-checkpoint=1 --backend local --config "$CUR/local-error.toml" --checkpoint-error-destroy="\`test\`.\`dup_detect\`" @@ -93,11 +93,11 @@ cleanup run_lightning --backend local --config "$CUR/local-limit-error-records.toml" --log-file "$LOG_FILE" run_sql "SELECT count(*) FROM test.dup_detect" check_contains "count(*): 174" -run_sql "SELECT count(*) FROM lightning_task_info.duplicate_records" +run_sql "SELECT count(*) FROM lightning_task_info.conflict_records" check_contains "count(*): 50" -run_sql "SELECT count(*) FROM lightning_task_info.duplicate_records WHERE error LIKE '%PRIMARY%'" +run_sql "SELECT count(*) FROM lightning_task_info.conflict_records WHERE error LIKE '%PRIMARY%'" check_contains "count(*): 49" -run_sql "SELECT count(*) FROM lightning_task_info.duplicate_records WHERE error LIKE '%uniq_col6_col7%'" +run_sql "SELECT count(*) FROM lightning_task_info.conflict_records WHERE error LIKE '%uniq_col6_col7%'" check_contains "count(*): 1" # 5. Test fail after duplicate detection. @@ -112,6 +112,6 @@ run_lightning_ctl --enable-checkpoint=1 --backend local --config "$CUR/local-rep run_lightning --enable-checkpoint=1 --backend local --config "$CUR/local-replace.toml" --log-file "$LOG_FILE" run_sql "SELECT count(*) FROM test.dup_detect" check_contains "count(*): 174" -run_sql "SELECT count(*) FROM lightning_task_info.duplicate_records" +run_sql "SELECT count(*) FROM lightning_task_info.conflict_records" check_contains "count(*): 227" check_not_contains "duplicate detection start" "$LOG_FILE"