Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

lightning: rename conflict table to conflict_records #45603

Merged
merged 2 commits into from
Jul 27, 2023
Merged

lightning: rename conflict table to conflict_records #45603

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5f7cfb7
lightning: rename conflict table to conflict_records
lance6716 Jul 27, 2023
5c37bf3
fix UT
lance6716 Jul 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion br/pkg/lightning/backend/tidb/tidb_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -690,7 +690,7 @@ func TestWriteRowsRecordOneError(t *testing.T) {
ExpectExec("\\QINSERT INTO `foo`.`bar`(`a`) VALUES(2)\\E").
WillReturnError(dupErr)
s.mockDB.
ExpectExec("INSERT INTO `tidb_lightning_errors`\\.duplicate_records.*").
ExpectExec("INSERT INTO `tidb_lightning_errors`\\.conflict_records.*").
WithArgs(sqlmock.AnyArg(), "`foo`.`bar`", "8.csv", int64(0), dupErr.Error(), 0, "(2)").
WillReturnResult(driver.ResultNoRows)

Expand Down
2 changes: 1 addition & 1 deletion br/pkg/lightning/errormanager/errormanager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ const (
// ConflictErrorTableName is the table name for duplicate detection.
ConflictErrorTableName = "conflict_error_v1"
// DupRecordTable is the table name to record duplicate data that displayed to user.
DupRecordTable = "duplicate_records"
DupRecordTable = "conflict_records"

createSyntaxErrorTable = `
CREATE TABLE IF NOT EXISTS %s.` + syntaxErrorTableName + ` (
Expand Down
18 changes: 9 additions & 9 deletions br/pkg/lightning/errormanager/errormanager_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func TestInit(t *testing.T) {
WillReturnResult(sqlmock.NewResult(6, 1))
mock.ExpectExec("CREATE TABLE IF NOT EXISTS `lightning_errors`\\.conflict_error_v1.*").
WillReturnResult(sqlmock.NewResult(7, 1))
mock.ExpectExec("CREATE TABLE IF NOT EXISTS `lightning_errors`\\.duplicate_records.*").
mock.ExpectExec("CREATE TABLE IF NOT EXISTS `lightning_errors`\\.conflict_records.*").
WillReturnResult(sqlmock.NewResult(7, 1))
err = em.Init(ctx)
require.NoError(t, err)
Expand Down Expand Up @@ -291,13 +291,13 @@ func TestErrorMgrErrorOutput(t *testing.T) {
output = em.Output()
expected = "\n" +
"Import Data Error Summary: \n" +
"+---+---------------------+-------------+----------------------------------+\n" +
"| # | ERROR TYPE | ERROR COUNT | ERROR DATA TABLE |\n" +
"+---+---------------------+-------------+----------------------------------+\n" +
"|\x1b[31m 1 \x1b[0m|\x1b[31m Data Type \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`type_error_v1` \x1b[0m|\n" +
"|\x1b[31m 2 \x1b[0m|\x1b[31m Data Syntax \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`syntax_error_v1` \x1b[0m|\n" +
"|\x1b[31m 3 \x1b[0m|\x1b[31m Charset Error \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m \x1b[0m|\n" +
"|\x1b[31m 4 \x1b[0m|\x1b[31m Unique Key Conflict \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`duplicate_records` \x1b[0m|\n" +
"+---+---------------------+-------------+----------------------------------+\n"
"+---+---------------------+-------------+---------------------------------+\n" +
"| # | ERROR TYPE | ERROR COUNT | ERROR DATA TABLE |\n" +
"+---+---------------------+-------------+---------------------------------+\n" +
"|\x1b[31m 1 \x1b[0m|\x1b[31m Data Type \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`type_error_v1` \x1b[0m|\n" +
"|\x1b[31m 2 \x1b[0m|\x1b[31m Data Syntax \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`syntax_error_v1` \x1b[0m|\n" +
"|\x1b[31m 3 \x1b[0m|\x1b[31m Charset Error \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m \x1b[0m|\n" +
"|\x1b[31m 4 \x1b[0m|\x1b[31m Unique Key Conflict \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`conflict_records` \x1b[0m|\n" +
"+---+---------------------+-------------+---------------------------------+\n"
require.Equal(t, expected, output)
}
18 changes: 9 additions & 9 deletions br/tests/lightning_config_max_error/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,20 +80,20 @@ check_contains "COUNT(*): ${duplicated_row_count}"
run_sql 'SELECT COUNT(*) FROM mytest.testtbl'
check_contains "COUNT(*): ${uniq_row_count}"

# Check tidb backend record duplicate entry in duplicate_records table
run_sql 'DROP TABLE IF EXISTS lightning_task_info.duplicate_records'
# Check tidb backend record duplicate entry in conflict_records table
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_records'
run_lightning --backend tidb --config "${mydir}/tidb.toml"
run_sql 'SELECT COUNT(*) FROM lightning_task_info.duplicate_records'
run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_records'
check_contains "COUNT(*): 10"
run_sql 'SELECT * FROM lightning_task_info.duplicate_records WHERE offset = 149'
run_sql 'SELECT * FROM lightning_task_info.conflict_records WHERE offset = 149'
check_contains "error: Error 1062 (23000): Duplicate entry '5' for key 'testtbl.PRIMARY'"
check_contains "row_data: ('5','bbb05')"

# Check max-error-record can limit the size of duplicate_records table
# Check max-error-record can limit the size of conflict_records table
run_sql 'DROP DATABASE IF EXISTS lightning_task_info'
run_sql 'DROP DATABASE IF EXISTS mytest'
run_lightning --backend tidb --config "${mydir}/tidb-limit-record.toml" 2>&1 | grep "\`lightning_task_info\`.\`duplicate_records\`" | grep -q "5"
run_sql 'SELECT COUNT(*) FROM lightning_task_info.duplicate_records'
run_lightning --backend tidb --config "${mydir}/tidb-limit-record.toml" 2>&1 | grep "\`lightning_task_info\`.\`conflict_records\`" | grep -q "5"
run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_records'
check_contains "COUNT(*): 1"

# Check conflict.threshold
Expand All @@ -109,8 +109,8 @@ run_sql 'DROP DATABASE IF EXISTS mytest'
rm "${TEST_DIR}/lightning.log"
run_lightning --backend tidb --config "${mydir}/tidb-error.toml" 2>&1 | grep -q "Error 1062 (23000): Duplicate entry '1' for key 'testtbl.PRIMARY'"
check_contains "Error 1062 (23000): Duplicate entry '1' for key 'testtbl.PRIMARY'" "${TEST_DIR}/lightning.log"
run_sql 'SELECT COUNT(*) FROM lightning_task_info.duplicate_records'
run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_records'
check_contains "COUNT(*): 1"
run_sql 'SELECT * FROM lightning_task_info.duplicate_records'
run_sql 'SELECT * FROM lightning_task_info.conflict_records'
check_contains "error: Error 1062 (23000): Duplicate entry '1' for key 'testtbl.PRIMARY'"
check_contains "row_data: ('1','bbb01')"
22 changes: 11 additions & 11 deletions br/tests/lightning_duplicate_detection_new/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,14 @@ if [ "$expected_rows" != "$actual_rows" ] || [ "$expected_pks" != "$actual_pks"
echo "local backend replace strategy result is not equal to tidb backend"
exit 1
fi
run_sql "SELECT count(*) FROM lightning_task_info.duplicate_records"
run_sql "SELECT count(*) FROM lightning_task_info.conflict_records"
check_contains "count(*): 227"
run_sql "SELECT count(*) FROM lightning_task_info.duplicate_records WHERE error = ''"
run_sql "SELECT count(*) FROM lightning_task_info.conflict_records WHERE error = ''"
check_contains "count(*): 0"
run_sql "SELECT * FROM lightning_task_info.duplicate_records WHERE row_id = 12"
run_sql "SELECT * FROM lightning_task_info.conflict_records WHERE row_id = 12"
check_contains "(171,'yRxZE',9201592769833450947,'xs3d',5,4,283270321)"
check_contains "[kv:1062]Duplicate entry '171' for key 'dup_detect.PRIMARY'"
run_sql "SELECT * FROM lightning_task_info.duplicate_records WHERE row_id = 1"
run_sql "SELECT * FROM lightning_task_info.conflict_records WHERE row_id = 1"
check_contains "(87,'nEoKu',7836621565948506759,'y6',48,0,177543185)"
check_contains "[kv:1062]Duplicate entry '0-177543185' for key 'dup_detect.uniq_col6_col7'"

Expand All @@ -65,18 +65,18 @@ if [ "$expected_rows" != "$actual_rows" ] || [ "$expected_pks" != "$actual_pks"
echo "local backend ignore strategy result is not equal to tidb backend"
exit 1
fi
run_sql "SELECT count(*) FROM lightning_task_info.duplicate_records"
run_sql "SELECT count(*) FROM lightning_task_info.conflict_records"
check_contains "count(*): 228"

# 3. Test error strategy.
cleanup
run_lightning --backend local --config "$CUR/local-error.toml" --log-file "$LOG_FILE" 2>&1 | grep -q "duplicate key in table \`test\`.\`dup_detect\` caused by index .*, but because checkpoint is off we can't have more details"
grep -q "duplicate key in table \`test\`.\`dup_detect\` caused by index .*, but because checkpoint is off we can't have more details" "$LOG_FILE"
run_sql "SELECT * FROM lightning_task_info.duplicate_records"
run_sql "SELECT * FROM lightning_task_info.conflict_records"
check_contains "error: duplicate key in table \`test\`.\`dup_detect\`"
run_lightning --backend local --config "$CUR/local-error.toml" --log-file "$LOG_FILE" --enable-checkpoint=1 2>&1 | grep -q "duplicate entry for key 'uniq_col6_col7', a pair of conflicting rows are (row 1 counting from offset 0 in file test.dup_detect.1.sql, row 101 counting from offset 0 in file test.dup_detect.4.sql)"
grep -q "duplicate entry for key 'uniq_col6_col7', a pair of conflicting rows are (row 1 counting from offset 0 in file test.dup_detect.1.sql, row 101 counting from offset 0 in file test.dup_detect.4.sql)" "$LOG_FILE"
run_sql "SELECT * FROM lightning_task_info.duplicate_records"
run_sql "SELECT * FROM lightning_task_info.conflict_records"
check_contains "error: duplicate entry for key 'uniq_col6_col7', a pair of conflicting rows are"
check_contains "restore table \`test\`.\`dup_detect\` failed: duplicate entry for key 'uniq_col6_col7', a pair of conflicting rows are (row 1 counting from offset 0 in file test.dup_detect.1.sql, row 101 counting from offset 0 in file test.dup_detect.4.sql)" "$LOG_FILE"
run_lightning_ctl --enable-checkpoint=1 --backend local --config "$CUR/local-error.toml" --checkpoint-error-destroy="\`test\`.\`dup_detect\`"
Expand All @@ -93,11 +93,11 @@ cleanup
run_lightning --backend local --config "$CUR/local-limit-error-records.toml" --log-file "$LOG_FILE"
run_sql "SELECT count(*) FROM test.dup_detect"
check_contains "count(*): 174"
run_sql "SELECT count(*) FROM lightning_task_info.duplicate_records"
run_sql "SELECT count(*) FROM lightning_task_info.conflict_records"
check_contains "count(*): 50"
run_sql "SELECT count(*) FROM lightning_task_info.duplicate_records WHERE error LIKE '%PRIMARY%'"
run_sql "SELECT count(*) FROM lightning_task_info.conflict_records WHERE error LIKE '%PRIMARY%'"
check_contains "count(*): 49"
run_sql "SELECT count(*) FROM lightning_task_info.duplicate_records WHERE error LIKE '%uniq_col6_col7%'"
run_sql "SELECT count(*) FROM lightning_task_info.conflict_records WHERE error LIKE '%uniq_col6_col7%'"
check_contains "count(*): 1"

# 5. Test fail after duplicate detection.
Expand All @@ -112,6 +112,6 @@ run_lightning_ctl --enable-checkpoint=1 --backend local --config "$CUR/local-rep
run_lightning --enable-checkpoint=1 --backend local --config "$CUR/local-replace.toml" --log-file "$LOG_FILE"
run_sql "SELECT count(*) FROM test.dup_detect"
check_contains "count(*): 174"
run_sql "SELECT count(*) FROM lightning_task_info.duplicate_records"
run_sql "SELECT count(*) FROM lightning_task_info.conflict_records"
check_contains "count(*): 227"
check_not_contains "duplicate detection start" "$LOG_FILE"