Skip to content

Commit

Permalink
*: support the variable of default_collation_for_utf8mb4 and corres…
Browse files Browse the repository at this point in the history 
…ponding behaviors (#46370)

close #46371
  • Loading branch information
@zimulala
zimulala authored Sep 5, 2023
1 parent 19e888f commit 6e7f36b
Show file tree
Hide file tree
Showing 18 changed files with 282 additions and 56 deletions.
53 changes: 53 additions & 0 deletions ddl/db_integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4407,3 +4407,56 @@ func TestReorganizePartitionWarning(t *testing.T) {
tk.MustExec("alter table t reorganize partition p0 into (partition p01 values less than (10), partition p02 values less than (20));")
tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 The statistics of related partitions will be outdated after reorganizing partitions. Please use 'ANALYZE TABLE' statement if you want to update it now"))
}

func TestDefaultCollationForUTF8MB4(t *testing.T) {
store := testkit.CreateMockStore(t, mockstore.WithDDLChecker())
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")

tk.MustExec("set @@session.default_collation_for_utf8mb4='utf8mb4_general_ci';")
tk.MustExec("drop table if exists t1, t2, t3, t4")
tk.MustExec("create table t1 (b char(1) default null)")
tk.MustQuery("show create table t1").Check(testkit.Rows("t1 CREATE TABLE `t1` (\n" +
" `b` char(1) DEFAULT NULL\n" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))
tk.MustExec("create table t4 (b char(1) default null) engine=InnoDB default charset=utf8mb4")
tk.MustQuery("show create table t4").Check(testkit.Rows("t4 CREATE TABLE `t4` (\n" +
" `b` char(1) COLLATE utf8mb4_general_ci DEFAULT NULL\n" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci"))
// test alter table ... character set
tk.MustExec("create table t2 (b char(1) default null) engine=InnoDB default charset=utf8mb4 COLLATE utf8mb4_0900_ai_ci")
tk.MustQuery("show create table t2").Check(testkit.Rows("t2 CREATE TABLE `t2` (\n" +
" `b` char(1) COLLATE utf8mb4_0900_ai_ci DEFAULT NULL\n" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci"))
tk.MustExec("alter table t2 default character set utf8mb4;")
tk.MustQuery("show create table t2").Check(testkit.Rows("t2 CREATE TABLE `t2` (\n" +
" `b` char(1) COLLATE utf8mb4_0900_ai_ci DEFAULT NULL\n" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci"))
tk.MustExec("alter table t2 add column c char(1);")
tk.MustQuery("show create table t2").Check(testkit.Rows("t2 CREATE TABLE `t2` (\n" +
" `b` char(1) COLLATE utf8mb4_0900_ai_ci DEFAULT NULL,\n" +
" `c` char(1) COLLATE utf8mb4_general_ci DEFAULT NULL\n" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci"))
// test alter table ... convert to character set
tk.MustExec("create table t3 (b char(1) default null) engine=InnoDB default charset=utf8mb4 COLLATE utf8mb4_0900_ai_ci")
tk.MustQuery("show create table t3").Check(testkit.Rows("t3 CREATE TABLE `t3` (\n" +
" `b` char(1) COLLATE utf8mb4_0900_ai_ci DEFAULT NULL\n" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci"))
tk.MustExec("alter table t3 convert to character set utf8mb4;")
tk.MustQuery("show create table t3").Check(testkit.Rows("t3 CREATE TABLE `t3` (\n" +
" `b` char(1) COLLATE utf8mb4_general_ci DEFAULT NULL\n" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci"))

// test database character set
tk.MustExec("drop database if exists dbx;")
tk.MustExec("drop database if exists dby;")
tk.MustExec("create database dbx DEFAULT CHARSET=utf8mb4;")
tk.MustQuery("show create database dbx").Check(testkit.Rows(
"dbx CREATE DATABASE `dbx` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci */"))
tk.MustExec("create database dby DEFAULT CHARSET=utf8mb4 COLLATE utf8mb4_0900_ai_ci;")
tk.MustQuery("show create database dby").Check(testkit.Rows(
"dby CREATE DATABASE `dby` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */"))
tk.MustExec("ALTER DATABASE dby CHARACTER SET = 'utf8mb4'")
tk.MustQuery("show create database dby").Check(testkit.Rows(
"dby CREATE DATABASE `dby` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci */"))
}
100 changes: 76 additions & 24 deletions ddl/ddl_api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,17 @@ func (d *ddl) CreateSchema(ctx sessionctx.Context, stmt *ast.CreateDatabaseStmt)
}
}
}
if !explicitCollation && explicitCharset {
coll, err := getDefaultCollationForUTF8MB4(ctx.GetSessionVars(), charsetOpt.Chs)
if err != nil {
return err
}
if len(coll) != 0 {
charsetOpt.Col = coll
}
}
dbInfo := &model.DBInfo{Name: stmt.Name}
chs, coll, err := ResolveCharsetCollation(charsetOpt)
chs, coll, err := ResolveCharsetCollation(ctx.GetSessionVars(), charsetOpt)
if err != nil {
return errors.Trace(err)
}
Expand Down Expand Up @@ -222,7 +231,7 @@ func (d *ddl) CreateSchemaWithInfo(

func (d *ddl) ModifySchemaCharsetAndCollate(ctx sessionctx.Context, stmt *ast.AlterDatabaseStmt, toCharset, toCollate string) (err error) {
if toCollate == "" {
if toCollate, err = charset.GetDefaultCollation(toCharset); err != nil {
if toCollate, err = GetDefaultCollation(ctx.GetSessionVars(), toCharset); err != nil {
return errors.Trace(err)
}
}
Expand Down Expand Up @@ -775,10 +784,38 @@ func buildColumnsAndConstraints(
return cols, constraints, nil
}

func getDefaultCollationForUTF8MB4(sessVars *variable.SessionVars, cs string) (string, error) {
if sessVars == nil || cs != charset.CharsetUTF8MB4 {
return "", nil
}
defaultCollation, err := sessVars.GetSessionOrGlobalSystemVar(context.Background(), variable.DefaultCollationForUTF8MB4)
if err != nil {
return "", err
}
return defaultCollation, nil
}

// GetDefaultCollation returns the default collation for charset and handle the default collation for UTF8MB4.
func GetDefaultCollation(sessVars *variable.SessionVars, cs string) (string, error) {
coll, err := getDefaultCollationForUTF8MB4(sessVars, cs)
if err != nil {
return "", errors.Trace(err)
}
if coll != "" {
return coll, nil
}

coll, err = charset.GetDefaultCollation(cs)
if err != nil {
return "", errors.Trace(err)
}
return coll, nil
}

// ResolveCharsetCollation will resolve the charset and collate by the order of parameters:
// * If any given ast.CharsetOpt is not empty, the resolved charset and collate will be returned.
// * If all ast.CharsetOpts are empty, the default charset and collate will be returned.
func ResolveCharsetCollation(charsetOpts ...ast.CharsetOpt) (chs string, coll string, err error) {
func ResolveCharsetCollation(sessVars *variable.SessionVars, charsetOpts ...ast.CharsetOpt) (chs string, coll string, err error) {
for _, v := range charsetOpts {
if v.Col != "" {
collation, err := collate.GetCollationByName(v.Col)
Expand All @@ -791,14 +828,21 @@ func ResolveCharsetCollation(charsetOpts ...ast.CharsetOpt) (chs string, coll st
return collation.CharsetName, v.Col, nil
}
if v.Chs != "" {
coll, err := charset.GetDefaultCollation(v.Chs)
coll, err := GetDefaultCollation(sessVars, v.Chs)
if err != nil {
return "", "", errors.Trace(err)
}
return v.Chs, coll, err
return v.Chs, coll, nil
}
}
chs, coll = charset.GetDefaultCharsetAndCollate()
utf8mb4Coll, err := getDefaultCollationForUTF8MB4(sessVars, chs)
if err != nil {
return "", "", errors.Trace(err)
}
if utf8mb4Coll != "" {
return chs, utf8mb4Coll, nil
}
return chs, coll, nil
}

Expand All @@ -807,14 +851,14 @@ func ResolveCharsetCollation(charsetOpts ...ast.CharsetOpt) (chs string, coll st
// CREATE TABLE t (a VARCHAR(255) BINARY) CHARSET utf8 COLLATE utf8_general_ci;
//
// The 'BINARY' sets the column collation to *_bin according to the table charset.
func OverwriteCollationWithBinaryFlag(colDef *ast.ColumnDef, chs, coll string) (newChs string, newColl string) {
func OverwriteCollationWithBinaryFlag(sessVars *variable.SessionVars, colDef *ast.ColumnDef, chs, coll string) (newChs string, newColl string) {
ignoreBinFlag := colDef.Tp.GetCharset() != "" && (colDef.Tp.GetCollate() != "" || containsColumnOption(colDef, ast.ColumnOptionCollate))
if ignoreBinFlag {
return chs, coll
}
needOverwriteBinColl := types.IsString(colDef.Tp.GetType()) && mysql.HasBinaryFlag(colDef.Tp.GetFlag())
if needOverwriteBinColl {
newColl, err := charset.GetDefaultCollation(chs)
newColl, err := GetDefaultCollation(sessVars, chs)
if err != nil {
return chs, coll
}
Expand Down Expand Up @@ -898,15 +942,15 @@ func buildColumnAndConstraint(
}

// specifiedCollate refers to the last collate specified in colDef.Options.
chs, coll, err := getCharsetAndCollateInColumnDef(colDef)
chs, coll, err := getCharsetAndCollateInColumnDef(ctx.GetSessionVars(), colDef)
if err != nil {
return nil, nil, errors.Trace(err)
}
chs, coll, err = ResolveCharsetCollation(
chs, coll, err = ResolveCharsetCollation(ctx.GetSessionVars(),
ast.CharsetOpt{Chs: chs, Col: coll},
ast.CharsetOpt{Chs: tblCharset, Col: tblCollate},
)
chs, coll = OverwriteCollationWithBinaryFlag(colDef, chs, coll)
chs, coll = OverwriteCollationWithBinaryFlag(ctx.GetSessionVars(), colDef, chs, coll)
if err != nil {
return nil, nil, errors.Trace(err)
}
Expand Down Expand Up @@ -2378,11 +2422,11 @@ func BuildSessionTemporaryTableInfo(ctx sessionctx.Context, is infoschema.InfoSc
// BuildTableInfoWithStmt builds model.TableInfo from a SQL statement without validity check
func BuildTableInfoWithStmt(ctx sessionctx.Context, s *ast.CreateTableStmt, dbCharset, dbCollate string, placementPolicyRef *model.PolicyRefInfo) (*model.TableInfo, error) {
colDefs := s.Cols
tableCharset, tableCollate, err := GetCharsetAndCollateInTableOption(0, s.Options)
tableCharset, tableCollate, err := GetCharsetAndCollateInTableOption(ctx.GetSessionVars(), 0, s.Options)
if err != nil {
return nil, errors.Trace(err)
}
tableCharset, tableCollate, err = ResolveCharsetCollation(
tableCharset, tableCollate, err = ResolveCharsetCollation(ctx.GetSessionVars(),
ast.CharsetOpt{Chs: tableCharset, Col: tableCollate},
ast.CharsetOpt{Chs: dbCharset, Col: dbCollate},
)
Expand Down Expand Up @@ -3386,11 +3430,11 @@ func isIgnorableSpec(tp ast.AlterTableType) bool {

// getCharsetAndCollateInColumnDef will iterate collate in the options, validate it by checking the charset
// of column definition. If there's no collate in the option, the default collate of column's charset will be used.
func getCharsetAndCollateInColumnDef(def *ast.ColumnDef) (chs, coll string, err error) {
func getCharsetAndCollateInColumnDef(sessVars *variable.SessionVars, def *ast.ColumnDef) (chs, coll string, err error) {
chs = def.Tp.GetCharset()
coll = def.Tp.GetCollate()
if chs != "" && coll == "" {
if coll, err = charset.GetDefaultCollation(chs); err != nil {
if coll, err = GetDefaultCollation(sessVars, chs); err != nil {
return "", "", errors.Trace(err)
}
}
Expand All @@ -3414,7 +3458,7 @@ func getCharsetAndCollateInColumnDef(def *ast.ColumnDef) (chs, coll string, err
// GetCharsetAndCollateInTableOption will iterate the charset and collate in the options,
// and returns the last charset and collate in options. If there is no charset in the options,
// the returns charset will be "", the same as collate.
func GetCharsetAndCollateInTableOption(startIdx int, options []*ast.TableOption) (chs, coll string, err error) {
func GetCharsetAndCollateInTableOption(sessVars *variable.SessionVars, startIdx int, options []*ast.TableOption) (chs, coll string, err error) {
for i := startIdx; i < len(options); i++ {
opt := options[i]
// we set the charset to the last option. example: alter table t charset latin1 charset utf8 collate utf8_bin;
Expand All @@ -3431,7 +3475,15 @@ func GetCharsetAndCollateInTableOption(startIdx int, options []*ast.TableOption)
return "", "", dbterror.ErrConflictingDeclarations.GenWithStackByArgs(chs, info.Name)
}
if len(coll) == 0 {
coll = info.DefaultCollation
defaultColl, err := getDefaultCollationForUTF8MB4(sessVars, chs)
if err != nil {
return "", "", errors.Trace(err)
}
if len(defaultColl) == 0 {
coll = info.DefaultCollation
} else {
coll = defaultColl
}
}
case ast.TableOptionCollate:
info, err := collate.GetCollationByName(opt.StrValue)
Expand Down Expand Up @@ -3702,7 +3754,7 @@ func (d *ddl) AlterTable(ctx context.Context, sctx sessionctx.Context, stmt *ast
continue
}
var toCharset, toCollate string
toCharset, toCollate, err = GetCharsetAndCollateInTableOption(i, spec.Options)
toCharset, toCollate, err = GetCharsetAndCollateInTableOption(sctx.GetSessionVars(), i, spec.Options)
if err != nil {
return err
}
Expand Down Expand Up @@ -4022,7 +4074,7 @@ func CreateNewColumn(ctx sessionctx.Context, schema *model.DBInfo, spec *ast.Alt
}
}

tableCharset, tableCollate, err := ResolveCharsetCollation(
tableCharset, tableCollate, err := ResolveCharsetCollation(ctx.GetSessionVars(),
ast.CharsetOpt{Chs: t.Meta().Charset, Col: t.Meta().Collate},
ast.CharsetOpt{Chs: schema.Charset, Col: schema.Collate},
)
Expand Down Expand Up @@ -5421,16 +5473,16 @@ func GetModifiableColumnJob(
chs = col.FieldType.GetCharset()
coll = col.FieldType.GetCollate()
} else {
chs, coll, err = getCharsetAndCollateInColumnDef(specNewColumn)
chs, coll, err = getCharsetAndCollateInColumnDef(sctx.GetSessionVars(), specNewColumn)
if err != nil {
return nil, errors.Trace(err)
}
chs, coll, err = ResolveCharsetCollation(
chs, coll, err = ResolveCharsetCollation(sctx.GetSessionVars(),
ast.CharsetOpt{Chs: chs, Col: coll},
ast.CharsetOpt{Chs: t.Meta().Charset, Col: t.Meta().Collate},
ast.CharsetOpt{Chs: schema.Charset, Col: schema.Collate},
)
chs, coll = OverwriteCollationWithBinaryFlag(specNewColumn, chs, coll)
chs, coll = OverwriteCollationWithBinaryFlag(sctx.GetSessionVars(), specNewColumn, chs, coll)
if err != nil {
return nil, errors.Trace(err)
}
Expand Down Expand Up @@ -6073,8 +6125,8 @@ func (d *ddl) AlterTableCharsetAndCollate(ctx sessionctx.Context, ident ast.Iden
}

if toCollate == "" {
// get the default collation of the charset.
toCollate, err = charset.GetDefaultCollation(toCharset)
// Get the default collation of the charset.
toCollate, err = GetDefaultCollation(ctx.GetSessionVars(), toCharset)
if err != nil {
return errors.Trace(err)
}
Expand Down Expand Up @@ -6417,7 +6469,7 @@ func checkAlterTableCharset(tblInfo *model.TableInfo, dbInfo *model.DBInfo, toCh
}

// This DDL will update the table charset to default charset.
origCharset, origCollate, err = ResolveCharsetCollation(
origCharset, origCollate, err = ResolveCharsetCollation(nil,
ast.CharsetOpt{Chs: origCharset, Col: origCollate},
ast.CharsetOpt{Chs: dbInfo.Charset, Col: dbInfo.Collate},
)
Expand Down
4 changes: 2 additions & 2 deletions ddl/schematracker/checker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,8 @@ func (d *Checker) Enable() {
}

// CreateTestDB creates a `test` database like the default behaviour of TiDB.
func (d *Checker) CreateTestDB() {
d.tracker.CreateTestDB()
func (d *Checker) CreateTestDB(ctx sessionctx.Context) {
d.tracker.CreateTestDB(ctx)
}

func (d *Checker) checkDBInfo(ctx sessionctx.Context, dbName model.CIStr) {
Expand Down
18 changes: 11 additions & 7 deletions ddl/schematracker/dm_tracker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,11 @@ func (d SchemaTracker) CreateSchema(ctx sessionctx.Context, stmt *ast.CreateData
}
}

chs, coll, err := ddl.ResolveCharsetCollation(charsetOpt)
var sessVars *variable.SessionVars
if ctx != nil {
sessVars = ctx.GetSessionVars()
}
chs, coll, err := ddl.ResolveCharsetCollation(sessVars, charsetOpt)
if err != nil {
return errors.Trace(err)
}
Expand All @@ -90,8 +94,8 @@ func (d SchemaTracker) CreateSchema(ctx sessionctx.Context, stmt *ast.CreateData
}

// CreateTestDB creates the `test` database, which is the default behavior of TiDB.
func (d SchemaTracker) CreateTestDB() {
_ = d.CreateSchema(nil, &ast.CreateDatabaseStmt{
func (d SchemaTracker) CreateTestDB(ctx sessionctx.Context) {
_ = d.CreateSchema(ctx, &ast.CreateDatabaseStmt{
Name: model.NewCIStr("test"),
})
}
Expand All @@ -111,7 +115,7 @@ func (d SchemaTracker) CreateSchemaWithInfo(_ sessionctx.Context, dbInfo *model.
}

// AlterSchema implements the DDL interface.
func (d SchemaTracker) AlterSchema(_ sessionctx.Context, stmt *ast.AlterDatabaseStmt) (err error) {
func (d SchemaTracker) AlterSchema(ctx sessionctx.Context, stmt *ast.AlterDatabaseStmt) (err error) {
dbInfo := d.SchemaByName(stmt.Name)
if dbInfo == nil {
return infoschema.ErrDatabaseNotExists.GenWithStackByArgs(stmt.Name.O)
Expand Down Expand Up @@ -150,8 +154,8 @@ func (d SchemaTracker) AlterSchema(_ sessionctx.Context, stmt *ast.AlterDatabase
toCollate = info.Name
}
}
if toCharset == "" {
if toCollate, err = charset.GetDefaultCollation(toCharset); err != nil {
if toCollate == "" {
if toCollate, err = ddl.GetDefaultCollation(ctx.GetSessionVars(), toCharset); err != nil {
return errors.Trace(err)
}
}
Expand Down Expand Up @@ -980,7 +984,7 @@ func (d SchemaTracker) AlterTable(ctx context.Context, sctx sessionctx.Context,
continue
}
var toCharset, toCollate string
toCharset, toCollate, err = ddl.GetCharsetAndCollateInTableOption(i, spec.Options)
toCharset, toCollate, err = ddl.GetCharsetAndCollateInTableOption(sctx.GetSessionVars(), i, spec.Options)
if err != nil {
return err
}
Expand Down
Loading

0 comments on commit 6e7f36b

Please sign in to comment.