From 515986ecddc8be373ff4e314f092a6901936795a Mon Sep 17 00:00:00 2001 From: Spade A <71589810+SpadeA-Tang@users.noreply.github.com> Date: Thu, 16 Jun 2022 21:36:36 +0800 Subject: [PATCH] cherry pick #35222 to release-5.4 Signed-off-by: ti-srebot --- errno/errcode.go | 1 + errno/errname.go | 1 + errors.toml | 5 ++ executor/errors.go | 1 + executor/load_data.go | 37 ++++++++++ server/server_test.go | 138 +++++++++++++++++++++++++++++++++++++ server/tidb_serial_test.go | 1 + 7 files changed, 184 insertions(+) diff --git a/errno/errcode.go b/errno/errcode.go index 046a7614bdb32..91d3e21789100 100644 --- a/errno/errcode.go +++ b/errno/errcode.go @@ -975,6 +975,7 @@ const ( ErrWarnOptimizerHintParseError = 8064 ErrWarnOptimizerHintInvalidInteger = 8065 ErrUnsupportedSecondArgumentType = 8066 + ErrColumnNotMatched = 8067 ErrInvalidPluginID = 8101 ErrInvalidPluginManifest = 8102 ErrInvalidPluginName = 8103 diff --git a/errno/errname.go b/errno/errname.go index 5163dab1f11bd..9cce3f0a8df1c 100644 --- a/errno/errname.go +++ b/errno/errname.go @@ -1038,6 +1038,7 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{ ErrInvalidWildCard: mysql.Message("Wildcard fields without any table name appears in wrong place", nil), ErrMixOfGroupFuncAndFieldsIncompatible: mysql.Message("In aggregated query without GROUP BY, expression #%d of SELECT list contains nonaggregated column '%s'; this is incompatible with sql_mode=only_full_group_by", nil), ErrUnsupportedSecondArgumentType: mysql.Message("JSON_OBJECTAGG: unsupported second argument type %v", nil), + ErrColumnNotMatched: mysql.Message("Load data: unmatched columns", nil), ErrLockExpire: mysql.Message("TTL manager has timed out, pessimistic locks may expire, please commit or rollback this transaction", nil), ErrTableOptionUnionUnsupported: mysql.Message("CREATE/ALTER table with union option is not supported", nil), ErrTableOptionInsertMethodUnsupported: mysql.Message("CREATE/ALTER table with insert method option is not supported", nil), diff --git a/errors.toml b/errors.toml index 229ad4df3847c..83390510861e9 100644 --- a/errors.toml +++ b/errors.toml @@ -861,6 +861,11 @@ error = ''' TiDB admin check table failed. ''' +["executor:8067"] +error = ''' +Load data: unmatched columns +''' + ["executor:8114"] error = ''' Unknown plan diff --git a/executor/errors.go b/executor/errors.go index f6e0f87d08e6b..5aeea2c4da092 100644 --- a/executor/errors.go +++ b/executor/errors.go @@ -32,6 +32,7 @@ var ( ErrUnsupportedPs = dbterror.ClassExecutor.NewStd(mysql.ErrUnsupportedPs) ErrSubqueryMoreThan1Row = dbterror.ClassExecutor.NewStd(mysql.ErrSubqueryNo1Row) ErrIllegalGrantForTable = dbterror.ClassExecutor.NewStd(mysql.ErrIllegalGrantForTable) + ErrColumnsNotMatched = dbterror.ClassExecutor.NewStd(mysql.ErrColumnNotMatched) ErrCantCreateUserWithGrant = dbterror.ClassExecutor.NewStd(mysql.ErrCantCreateUserWithGrant) ErrPasswordNoMatch = dbterror.ClassExecutor.NewStd(mysql.ErrPasswordNoMatch) diff --git a/executor/load_data.go b/executor/load_data.go index 7d124c0cacf3d..5d65e01595ccb 100644 --- a/executor/load_data.go +++ b/executor/load_data.go @@ -131,6 +131,36 @@ type FieldMapping struct { UserVar *ast.VariableExpr } +// reorderColumns reorder the e.insertColumns according to the order of columnNames +// Note: We must ensure there must be one-to-one mapping between e.insertColumns and columnNames in terms of column name. +func (e *LoadDataInfo) reorderColumns(columnNames []string) error { + cols := e.insertColumns + + if len(cols) != len(columnNames) { + return ErrColumnsNotMatched + } + + reorderedColumns := make([]*table.Column, len(cols)) + + if columnNames == nil { + return nil + } + + mapping := make(map[string]int) + for idx, colName := range columnNames { + mapping[strings.ToLower(colName)] = idx + } + + for _, col := range cols { + idx := mapping[col.Name.L] + reorderedColumns[idx] = col + } + + e.insertColumns = reorderedColumns + + return nil +} + // initLoadColumns sets columns which the input fields loaded to. func (e *LoadDataInfo) initLoadColumns(columnNames []string) error { var cols []*table.Column @@ -163,6 +193,13 @@ func (e *LoadDataInfo) initLoadColumns(columnNames []string) error { break } } + + // e.insertColumns is appended according to the original tables' column sequence. + // We have to reorder it to follow the use-specified column order which is shown in the columnNames. + if err = e.reorderColumns(columnNames); err != nil { + return err + } + e.rowLen = len(e.insertColumns) // Check column whether is specified only once. err = table.CheckOnce(cols) diff --git a/server/server_test.go b/server/server_test.go index 034587d7be1f7..f81d8ff5c9069 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -850,6 +850,144 @@ func (cli *testServerClient) checkRows(t *testing.T, rows *sql.Rows, expectedRow require.Equal(t, strings.Join(expectedRows, "\n"), strings.Join(result, "\n")) } +func (cli *testServerClient) runTestLoadDataWithColumnList(t *testing.T, _ *Server) { + fp, err := os.CreateTemp("", "load_data_test.csv") + require.NoError(t, err) + path := fp.Name() + require.NotNil(t, fp) + defer func() { + err = fp.Close() + require.NoError(t, err) + err = os.Remove(path) + require.NoError(t, err) + }() + + _, err = fp.WriteString("dsadasdas\n" + + "\"1\",\"1\",,\"2022-04-19\",\"a\",\"2022-04-19 00:00:01\"\n" + + "\"1\",\"2\",\"a\",\"2022-04-19\",\"a\",\"2022-04-19 00:00:01\"\n" + + "\"1\",\"3\",\"a\",\"2022-04-19\",\"a\",\"2022-04-19 00:00:01\"\n" + + "\"1\",\"4\",\"a\",\"2022-04-19\",\"a\",\"2022-04-19 00:00:01\"") + + cli.runTestsOnNewDB(t, func(config *mysql.Config) { + config.AllowAllFiles = true + config.Params["sql_mode"] = "''" + }, "LoadData", func(db *testkit.DBTestKit) { + db.MustExec("use test") + db.MustExec("drop table if exists t66") + db.MustExec("create table t66 (id int primary key,k int,c varchar(10),dt date,vv char(1),ts datetime)") + db.MustExec(fmt.Sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE t66 FIELDS TERMINATED BY ',' ENCLOSED BY '\\\"' IGNORE 1 LINES (k,id,c,dt,vv,ts)", path)) + rows := db.MustQuery("select * from t66") + var ( + id sql.NullString + k sql.NullString + c sql.NullString + dt sql.NullString + vv sql.NullString + ts sql.NullString + ) + columns := []*sql.NullString{&k, &id, &c, &dt, &vv, &ts} + require.Truef(t, rows.Next(), "unexpected data") + err := rows.Scan(&id, &k, &c, &dt, &vv, &ts) + require.NoError(t, err) + columnsAsExpected(t, columns, strings.Split("1,1,,2022-04-19,a,2022-04-19 00:00:01", ",")) + require.Truef(t, rows.Next(), "unexpected data") + err = rows.Scan(&id, &k, &c, &dt, &vv, &ts) + require.NoError(t, err) + columnsAsExpected(t, columns, strings.Split("1,2,a,2022-04-19,a,2022-04-19 00:00:01", ",")) + require.Truef(t, rows.Next(), "unexpected data") + err = rows.Scan(&id, &k, &c, &dt, &vv, &ts) + require.NoError(t, err) + columnsAsExpected(t, columns, strings.Split("1,3,a,2022-04-19,a,2022-04-19 00:00:01", ",")) + require.Truef(t, rows.Next(), "unexpected data") + err = rows.Scan(&id, &k, &c, &dt, &vv, &ts) + require.NoError(t, err) + columnsAsExpected(t, columns, strings.Split("1,4,a,2022-04-19,a,2022-04-19 00:00:01", ",")) + }) + + // Also test cases where column list only specifies partial columns + cli.runTestsOnNewDB(t, func(config *mysql.Config) { + config.AllowAllFiles = true + config.Params["sql_mode"] = "''" + }, "LoadData", func(db *testkit.DBTestKit) { + db.MustExec("use test") + db.MustExec("drop table if exists t66") + db.MustExec("create table t66 (id int primary key,k int,c varchar(10),dt date,vv char(1),ts datetime)") + db.MustExec(fmt.Sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE t66 FIELDS TERMINATED BY ',' ENCLOSED BY '\\\"' IGNORE 1 LINES (k,id,c)", path)) + rows := db.MustQuery("select * from t66") + var ( + id sql.NullString + k sql.NullString + c sql.NullString + dt sql.NullString + vv sql.NullString + ts sql.NullString + ) + columns := []*sql.NullString{&k, &id, &c, &dt, &vv, &ts} + require.Truef(t, rows.Next(), "unexpected data") + err = rows.Scan(&id, &k, &c, &dt, &vv, &ts) + require.NoError(t, err) + columnsAsExpected(t, columns, strings.Split("1,1,,,,", ",")) + require.Truef(t, rows.Next(), "unexpected data") + err = rows.Scan(&id, &k, &c, &dt, &vv, &ts) + require.NoError(t, err) + columnsAsExpected(t, columns, strings.Split("1,2,a,,,", ",")) + require.Truef(t, rows.Next(), "unexpected data") + err = rows.Scan(&id, &k, &c, &dt, &vv, &ts) + require.NoError(t, err) + columnsAsExpected(t, columns, strings.Split("1,3,a,,,", ",")) + require.Truef(t, rows.Next(), "unexpected data") + err = rows.Scan(&id, &k, &c, &dt, &vv, &ts) + require.NoError(t, err) + columnsAsExpected(t, columns, strings.Split("1,4,a,,,", ",")) + }) + + // Also test for case-insensitivity + cli.runTestsOnNewDB(t, func(config *mysql.Config) { + config.AllowAllFiles = true + config.Params["sql_mode"] = "''" + }, "LoadData", func(db *testkit.DBTestKit) { + db.MustExec("use test") + db.MustExec("drop table if exists t66") + db.MustExec("create table t66 (id int primary key,k int,c varchar(10),dt date,vv char(1),ts datetime)") + // We modify the upper case and lower case in the column list to test the case-insensitivity + db.MustExec(fmt.Sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE t66 FIELDS TERMINATED BY ',' ENCLOSED BY '\\\"' IGNORE 1 LINES (K,Id,c,dT,Vv,Ts)", path)) + rows := db.MustQuery("select * from t66") + var ( + id sql.NullString + k sql.NullString + c sql.NullString + dt sql.NullString + vv sql.NullString + ts sql.NullString + ) + columns := []*sql.NullString{&k, &id, &c, &dt, &vv, &ts} + require.Truef(t, rows.Next(), "unexpected data") + err := rows.Scan(&id, &k, &c, &dt, &vv, &ts) + require.NoError(t, err) + columnsAsExpected(t, columns, strings.Split("1,1,,2022-04-19,a,2022-04-19 00:00:01", ",")) + require.Truef(t, rows.Next(), "unexpected data") + err = rows.Scan(&id, &k, &c, &dt, &vv, &ts) + require.NoError(t, err) + columnsAsExpected(t, columns, strings.Split("1,2,a,2022-04-19,a,2022-04-19 00:00:01", ",")) + require.Truef(t, rows.Next(), "unexpected data") + err = rows.Scan(&id, &k, &c, &dt, &vv, &ts) + require.NoError(t, err) + columnsAsExpected(t, columns, strings.Split("1,3,a,2022-04-19,a,2022-04-19 00:00:01", ",")) + require.Truef(t, rows.Next(), "unexpected data") + err = rows.Scan(&id, &k, &c, &dt, &vv, &ts) + require.NoError(t, err) + columnsAsExpected(t, columns, strings.Split("1,4,a,2022-04-19,a,2022-04-19 00:00:01", ",")) + }) +} + +func columnsAsExpected(t *testing.T, columns []*sql.NullString, expected []string) { + require.Equal(t, len(columns), len(expected)) + + for i := 0; i < len(columns); i++ { + require.Equal(t, expected[i], columns[i].String) + } +} + func (cli *testServerClient) runTestLoadData(t *testing.T, server *Server) { // create a file and write data. path := "/tmp/load_data_test.csv" diff --git a/server/tidb_serial_test.go b/server/tidb_serial_test.go index 141681e1df24e..ec323fcaec0f3 100644 --- a/server/tidb_serial_test.go +++ b/server/tidb_serial_test.go @@ -41,6 +41,7 @@ func TestLoadData(t *testing.T) { ts, cleanup := createTidbTestSuite(t) defer cleanup() + ts.runTestLoadDataWithColumnList(t, ts.server) ts.runTestLoadData(t, ts.server) ts.runTestLoadDataWithSelectIntoOutfile(t, ts.server) ts.runTestLoadDataForSlowLog(t, ts.server)