expression: support GBK charset for builtin function upper()

cmd/explaintest/r/new_character_set_builtin.result

@@ -9,3 +9,14 @@ select hex(a), hex(b), hex(c) from t;
 hex(a)	hex(b)	hex(c)
 E4B880E4BA8CE4B889	D2BBB6FEC8FD	E4B880E4BA8CE4B8890000000000000000000000
 set @@tidb_enable_vectorized_expression = false;
+drop table if exists t;
+create table t (a char(100) charset utf8mb4, b char(100) charset gbk);
+insert into t values ('àáèéêìíòóùúüāēěīńňōūǎǐǒǔǖǘǚǜⅪⅫ', 'àáèéêìíòóùúüāēěīńňōūǎǐǒǔǖǘǚǜⅪⅫ');
+select upper(a), upper(b) from t;
+upper(a)	upper(b)
+ÀÁÈÉÊÌÍÒÓÙÚÜĀĒĚĪŃŇŌŪǍǏǑǓǕǗǙǛⅪⅫ	àáèéêìíòóùúüāēěīńňōūǎǐǒǔǖǘǚǜⅪⅫ
+set @@tidb_enable_vectorized_expression = true;
+select upper(a), upper(b) from t;
+upper(a)	upper(b)
+ÀÁÈÉÊÌÍÒÓÙÚÜĀĒĚĪŃŇŌŪǍǏǑǓǕǗǙǛⅪⅫ	àáèéêìíòóùúüāēěīńňōūǎǐǒǔǖǘǚǜⅪⅫ
+set @@tidb_enable_vectorized_expression = false;

cmd/explaintest/t/new_character_set_builtin.test

@@ -6,3 +6,13 @@ select hex(a), hex(b), hex(c) from t;
 set @@tidb_enable_vectorized_expression = true;
 select hex(a), hex(b), hex(c) from t;
 set @@tidb_enable_vectorized_expression = false;
+
+-- test for builtin function upper()
+drop table if exists t;
+create table t (a char(100) charset utf8mb4, b char(100) charset gbk);
+insert into t values ('àáèéêìíòóùúüāēěīńňōūǎǐǒǔǖǘǚǜⅪⅫ', 'àáèéêìíòóùúüāēěīńňōūǎǐǒǔǖǘǚǜⅪⅫ');
+select upper(a), upper(b) from t;
+set @@tidb_enable_vectorized_expression = true;
+select upper(a), upper(b) from t;
+set @@tidb_enable_vectorized_expression = false;
+

expression/builtin_string.go

@@ -877,19 +877,23 @@ func (c *upperFunctionClass) getFunction(ctx sessionctx.Context, args []Expressi
 		sig = &builtinUpperSig{bf}
 		sig.setPbCode(tipb.ScalarFuncSig_Upper)
 	} else {
-		sig = &builtinUpperUTF8Sig{bf}
+		sig = &builtinUpperUTF8Sig{bf, charset.NewEncoding(argTp.Charset)}
 		sig.setPbCode(tipb.ScalarFuncSig_UpperUTF8)
 	}
 	return sig, nil
 }
 
 type builtinUpperUTF8Sig struct {
 	baseBuiltinFunc
+	encoding *charset.Encoding
 }
 
 func (b *builtinUpperUTF8Sig) Clone() builtinFunc {
 	newSig := &builtinUpperUTF8Sig{}
 	newSig.cloneFrom(&b.baseBuiltinFunc)
+	if b.encoding != nil {
+		newSig.encoding = charset.NewEncoding(b.encoding.Name())
+	}
 	return newSig
 }
 
@@ -901,7 +905,7 @@ func (b *builtinUpperUTF8Sig) evalString(row chunk.Row) (d string, isNull bool,
 		return d, isNull, err
 	}
 
-	return strings.ToUpper(d), false, nil
+	return b.encoding.ToUpper(d), false, nil
 }
 
 type builtinUpperSig struct {

expression/builtin_string_test.go

@@ -598,6 +598,28 @@ func (s *testEvaluatorSuite) TestUpper(c *C) {
 
 	_, err := funcs[ast.Upper].getFunction(s.ctx, []Expression{varcharCon})
 	c.Assert(err, IsNil)
+
+	// Test GBK String
+	tbl := []struct {
+		input  string
+		chs    string
+		result string
+	}{
+		{"abc", "gbk", "ABC"},
+		{"一二三", "gbk", "一二三"},
+		{"àbc", "gbk", "àBC"},
+		{"àáèéêìíòóùúüāēěīńňōūǎǐǒǔǖǘǚǜⅪⅫ", "gbk", "àáèéêìíòóùúüāēěīńňōūǎǐǒǔǖǘǚǜⅪⅫ"},
+		{"àáèéêìíòóùúüāēěīńňōūǎǐǒǔǖǘǚǜⅪⅫ", "", "ÀÁÈÉÊÌÍÒÓÙÚÜĀĒĚĪŃŇŌŪǍǏǑǓǕǗǙǛⅪⅫ"},
+	}
+	for _, t := range tbl {
+		err := s.ctx.GetSessionVars().SetSystemVar(variable.CharacterSetConnection, t.chs)
+		c.Assert(err, IsNil)
+		f, err := newFunctionForTest(s.ctx, ast.Upper, s.primitiveValsToConstants([]interface{}{t.input})...)
+		c.Assert(err, IsNil)
+		d, err := f.Eval(chunk.Row{})
+		c.Assert(err, IsNil)
+		c.Assert(d.GetString(), Equals, t.result)
+	}
 }
 
 func (s *testEvaluatorSuite) TestReverse(c *C) {

expression/builtin_string_vec.go

@@ -141,7 +141,7 @@ func (b *builtinUpperUTF8Sig) vecEvalString(input *chunk.Chunk, result *chunk.Co
 	}
 
 	for i := 0; i < input.NumRows(); i++ {
-		result.SetRaw(i, []byte(strings.ToUpper(result.GetString(i))))
+		result.SetRaw(i, []byte(b.encoding.ToUpper(result.GetString(i))))
 	}
 	return nil
 }

parser/charset/encoding.go

@@ -17,6 +17,7 @@ import (
 	"bytes"
 	"fmt"
 	"strings"
+	"unicode"
 
 	"github.com/cznic/mathutil"
 	"github.com/pingcap/tidb/parser/mysql"
@@ -43,9 +44,10 @@ func Formatted(label string) EncodingLabel {
 
 // Encoding provide a interface to encode/decode a string with specific encoding.
 type Encoding struct {
-	enc        encoding.Encoding
-	name       string
-	charLength func([]byte) int
+	enc         encoding.Encoding
+	name        string
+	charLength  func([]byte) int
+	specialCase unicode.SpecialCase
 }
 
 // Enabled indicates whether the non-utf8 encoding is used.
@@ -66,9 +68,10 @@ func NewEncoding(label string) *Encoding {
 	e, name := Lookup(label)
 	if e != nil && name != encodingLegacy {
 		return &Encoding{
-			enc:        e,
-			name:       name,
-			charLength: FindNextCharacterLength(name),
+			enc:         e,
+			name:        name,
+			charLength:  FindNextCharacterLength(name),
+			specialCase: LookupSpecialCase(name),
 		}
 	}
 	return &Encoding{name: name}
@@ -85,6 +88,7 @@ func (e *Encoding) UpdateEncoding(label EncodingLabel) {
 		e.enc = nil
 		e.charLength = nil
 	}
+	e.specialCase = LookupSpecialCase(e.name)
 }
 
 // Encode convert bytes from utf-8 charset to a specific charset.

parser/charset/special_case_tables.go

@@ -0,0 +1,100 @@
+// Copyright 2021 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package charset
+
+import (
+	"strings"
+	"unicode"
+)
+
+func (e *Encoding) ToUpper(d string) string {
+	return strings.ToUpperSpecial(e.specialCase, d)
+}
+
+func LookupSpecialCase(label string) unicode.SpecialCase {
+	label = strings.ToLower(strings.Trim(label, "\t\n\r\f "))
+	return specailCases[label].c
+}
+
+var specailCases = map[string]struct {
+	c unicode.SpecialCase
+}{
+	"utf-8":          {nil},
+	"ibm866":         {nil},
+	"iso-8859-2":     {nil},
+	"iso-8859-3":     {nil},
+	"iso-8859-4":     {nil},
+	"iso-8859-5":     {nil},
+	"iso-8859-6":     {nil},
+	"iso-8859-7":     {nil},
+	"iso-8859-8":     {nil},
+	"iso-8859-8-i":   {nil},
+	"iso-8859-10":    {nil},
+	"iso-8859-13":    {nil},
+	"iso-8859-14":    {nil},
+	"iso-8859-15":    {nil},
+	"iso-8859-16":    {nil},
+	"koi8-r":         {nil},
+	"macintosh":      {nil},
+	"windows-874":    {nil},
+	"windows-1250":   {nil},
+	"windows-1251":   {nil},
+	"windows-1252":   {nil},
+	"windows-1253":   {nil},
+	"windows-1254":   {nil},
+	"windows-1255":   {nil},
+	"windows-1256":   {nil},
+	"windows-1257":   {nil},
+	"windows-1258":   {nil},
+	"x-mac-cyrillic": {nil},
+	"gbk":            {GBKCase},
+	"gb18030":        {nil},
+	"hz-gb-2312":     {nil},
+	"big5":           {nil},
+	"euc-jp":         {nil},
+	"iso-2022-jp":    {nil},
+	"shift_jis":      {nil},
+	"euc-kr":         {nil},
+	"replacement":    {nil},
+	"utf-16be":       {nil},
+	"utf-16le":       {nil},
+	"x-user-defined": {nil},
+}
+
+// follow https://dev.mysql.com/worklog/task/?id=4583 for GBK
+var GBKCase = unicode.SpecialCase{
+	unicode.CaseRange{0x00E0, 0x00E1, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x00E8, 0x00EA, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x00EC, 0x00ED, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x00F2, 0x00F3, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x00F9, 0x00FA, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x00FC, 0x00FC, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x0101, 0x0101, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x0113, 0x0113, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x011B, 0x011B, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x012B, 0x012B, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x0144, 0x0144, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x0148, 0x0148, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x014D, 0x014D, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x016B, 0x016B, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x01CE, 0x01CE, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x01D0, 0x01D0, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x01D2, 0x01D2, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x01D4, 0x01D4, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x01D6, 0x01D6, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x01D8, 0x01D8, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x01DA, 0x01DA, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x01DC, 0x01DC, [unicode.MaxCase]rune{0, 0, 0}},
+	unicode.CaseRange{0x216A, 0x216B, [unicode.MaxCase]rune{0, 0, 0}},
+}