Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

expression: support GBK for builtin function Decode and Encode #29315

Merged
merged 25 commits into from
Nov 3, 2021
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
4be11b3
expression: support GBK for builtin function Decode and Encode
hawkingrei Nov 1, 2021
143bedd
expression: support GBK for builtin function Decode and encode
hawkingrei Nov 2, 2021
23478d8
expression: support GBK for builtin function Decode and encode
hawkingrei Nov 2, 2021
bd55e1a
Merge branch 'master' into add_gbk_for_decode
hawkingrei Nov 2, 2021
8249213
update
hawkingrei Nov 2, 2021
48ecc6a
expression: support GBK for builtin function Decode and Encode
hawkingrei Nov 3, 2021
034b79a
expression: support GBK for builtin function Decode and encode
hawkingrei Nov 3, 2021
461a992
expression: support GBK for builtin function Decode and encode
hawkingrei Nov 3, 2021
27be1aa
expression: support GBK for builtin function Decode and Encode
hawkingrei Nov 3, 2021
d761d25
expression: support GBK for builtin function Decode and encode
hawkingrei Nov 3, 2021
915fd5c
expression: support GBK for builtin function Decode and encode
hawkingrei Nov 3, 2021
945cfa0
expression: support GBK for builtin function Decode and encode
hawkingrei Nov 3, 2021
b8ee640
Merge branch 'master' into add_gbk_for_decode
AilinKid Nov 3, 2021
e981114
expression: support GBK for builtin function Decode and encode
hawkingrei Nov 3, 2021
c56fc78
Merge branch 'add_gbk_for_decode' of github.com:hawkingrei/tidb into …
hawkingrei Nov 3, 2021
2885640
Merge branch 'master' of https://github.com/pingcap/tidb into add_gbk…
hawkingrei Nov 3, 2021
9db095b
Merge branch 'add_gbk_for_decode' of github.com:hawkingrei/tidb into …
hawkingrei Nov 3, 2021
2ec28d9
expression: support GBK for builtin function Decode and encode
hawkingrei Nov 3, 2021
4967cd7
Merge branch 'add_gbk_for_decode' of github.com:hawkingrei/tidb into …
hawkingrei Nov 3, 2021
f6b5899
expression: support GBK for builtin function Decode and encode
hawkingrei Nov 3, 2021
e09a734
Merge branch 'add_gbk_for_decode' of github.com:hawkingrei/tidb into …
hawkingrei Nov 3, 2021
ce4c9d0
expression: support GBK for builtin function Decode and Encode
hawkingrei Nov 3, 2021
0f22e3a
Merge branch 'master' into add_gbk_for_decode
AilinKid Nov 3, 2021
f5af85f
expression: support GBK for builtin function Decode and encode
hawkingrei Nov 3, 2021
8345da6
Merge branch 'master' into add_gbk_for_decode
ti-chi-bot Nov 3, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file modified cmd/explaintest/r/new_character_set_builtin.result
Binary file not shown.
6 changes: 6 additions & 0 deletions cmd/explaintest/t/new_character_set_builtin.test
Original file line number Diff line number Diff line change
Expand Up @@ -66,3 +66,9 @@ select md5(a), md5(b), md5(c) from t;
set @@tidb_enable_vectorized_expression = true;
select md5(a), md5(b), md5(c) from t;
set @@tidb_enable_vectorized_expression = false;

-- test for builtin function decode()/encode()
drop table if exists t;
create table t (a char(20) charset utf8mb4, b char(20) charset gbk, c binary(20));
insert into t values ('一二三', '一二三', '一二三');
select decode(encode(a,"monty"),"monty"), decode(encode(b,"monty"),"monty"), decode(encode(c,"monty"),"monty") from t;
hawkingrei marked this conversation as resolved.
Show resolved Hide resolved
24 changes: 22 additions & 2 deletions expression/builtin_encryption.go
Original file line number Diff line number Diff line change
Expand Up @@ -407,13 +407,23 @@ func (b *builtinDecodeSig) evalString(row chunk.Row) (string, bool, error) {
if isNull || err != nil {
return "", true, err
}
dataTp := b.args[0].GetType()
dataBuf, err := charset.NewEncoding(dataTp.Charset).EncodeString(dataStr)
if err != nil {
return "", true, err
}

passwordStr, isNull, err := b.args[1].EvalString(b.ctx, row)
if isNull || err != nil {
return "", true, err
}
passwordTp := b.args[1].GetType()
passwordBuf, err := charset.NewEncoding(passwordTp.Charset).EncodeString(passwordStr)
hawkingrei marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return "", true, err
}

decodeStr, err := encrypt.SQLDecode(dataStr, passwordStr)
decodeStr, err := encrypt.SQLDecode(dataBuf, passwordBuf)
return decodeStr, false, err
}

Expand Down Expand Up @@ -470,13 +480,23 @@ func (b *builtinEncodeSig) evalString(row chunk.Row) (string, bool, error) {
if isNull || err != nil {
return "", true, err
}
decodeTp := b.args[0].GetType()
decodeBuff, err := charset.NewEncoding(decodeTp.Charset).EncodeString(decodeStr)
if err != nil {
return "", true, err
hawkingrei marked this conversation as resolved.
Show resolved Hide resolved
}

passwordStr, isNull, err := b.args[1].EvalString(b.ctx, row)
if isNull || err != nil {
return "", true, err
}
passwordTp := b.args[1].GetType()
passwordBuff, err := charset.NewEncoding(passwordTp.Charset).EncodeString(passwordStr)
if err != nil {
return "", true, err
}

dataStr, err := encrypt.SQLEncode(decodeStr, passwordStr)
dataStr, err := encrypt.SQLEncode(decodeBuff, passwordBuff)
return dataStr, false, err
}

Expand Down
74 changes: 46 additions & 28 deletions expression/builtin_encryption_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,56 +33,74 @@ import (
)

var cryptTests = []struct {
chs string
origin interface{}
password interface{}
crypt interface{}
}{
{"", "", ""},
{"pingcap", "1234567890123456", "2C35B5A4ADF391"},
{"pingcap", "asdfjasfwefjfjkj", "351CC412605905"},
{"pingcap123", "123456789012345678901234", "7698723DC6DFE7724221"},
{"pingcap#%$%^", "*^%YTu1234567", "8634B9C55FF55E5B6328F449"},
{"pingcap", "", "4A77B524BD2C5C"},
{"分布式データベース", "pass1234@#$%%^^&", "80CADC8D328B3026D04FB285F36FED04BBCA0CC685BF78B1E687CE"},
{"分布式データベース", "分布式7782734adgwy1242", "0E24CFEF272EE32B6E0BFBDB89F29FB43B4B30DAA95C3F914444BC"},
{"pingcap", "密匙", "CE5C02A5010010"},
{"pingcap数据库", "数据库passwd12345667", "36D5F90D3834E30E396BE3226E3B4ED3"},
{"数据库5667", 123.435, "B22196D0569386237AE12F8AAB"},
{nil, "数据库passwd12345667", nil},
{mysql.DefaultCollationName, "", "", ""},
{mysql.DefaultCollationName, "pingcap", "1234567890123456", "2C35B5A4ADF391"},
{mysql.DefaultCollationName, "pingcap", "asdfjasfwefjfjkj", "351CC412605905"},
{mysql.DefaultCollationName, "pingcap123", "123456789012345678901234", "7698723DC6DFE7724221"},
{mysql.DefaultCollationName, "pingcap#%$%^", "*^%YTu1234567", "8634B9C55FF55E5B6328F449"},
{mysql.DefaultCollationName, "pingcap", "", "4A77B524BD2C5C"},
{mysql.DefaultCollationName, "分布式データベース", "pass1234@#$%%^^&", "80CADC8D328B3026D04FB285F36FED04BBCA0CC685BF78B1E687CE"},
{mysql.DefaultCollationName, "分布式データベース", "分布式7782734adgwy1242", "0E24CFEF272EE32B6E0BFBDB89F29FB43B4B30DAA95C3F914444BC"},
{mysql.DefaultCollationName, "pingcap", "密匙", "CE5C02A5010010"},
{"gbk", "pingcap", "密匙", "E407AC6F691ADE"},
{mysql.DefaultCollationName, "pingcap数据库", "数据库passwd12345667", "36D5F90D3834E30E396BE3226E3B4ED3"},
{"gbk", "pingcap数据库", "数据库passwd12345667", "B4BDBD6EC8346379F42836E2E0"},
{mysql.DefaultCollationName, "数据库5667", 123.435, "B22196D0569386237AE12F8AAB"},
{"gbk", "数据库5667", 123.435, "79E22979BD860EF58229"},
{mysql.DefaultCollationName, nil, "数据库passwd12345667", nil},
}

func TestSQLDecode(t *testing.T) {
t.Parallel()
ctx := createContext(t)
fc := funcs[ast.Decode]
for _, tt := range cryptTests {
str := types.NewDatum(tt.origin)
password := types.NewDatum(tt.password)

f, err := fc.getFunction(ctx, datumsToConstants([]types.Datum{str, password}))
err := ctx.GetSessionVars().SetSystemVar(variable.CharacterSetConnection, tt.chs)
require.NoError(t, err)
crypt, err := evalBuiltinFunc(f, chunk.Row{})
err = ctx.GetSessionVars().SetSystemVar(variable.CollationConnection, tt.chs)
require.NoError(t, err)
require.Equal(t, types.NewDatum(tt.crypt), toHex(crypt))
f, err := newFunctionForTest(ctx, ast.Decode, primitiveValsToConstants(ctx, []interface{}{tt.origin, tt.password})...)
require.NoError(t, err)
d, err := f.Eval(chunk.Row{})
require.NoError(t, err)
if !d.IsNull() {
d = toHex(d)
}
require.Equal(t, types.NewDatum(tt.crypt), d)
}
testNullInput(t, ctx, ast.Decode)
}

func TestSQLEncode(t *testing.T) {
t.Parallel()
ctx := createContext(t)

fc := funcs[ast.Encode]
for _, test := range cryptTests {
password := types.NewDatum(test.password)
cryptStr := fromHex(test.crypt)

f, err := fc.getFunction(ctx, datumsToConstants([]types.Datum{cryptStr, password}))
err := ctx.GetSessionVars().SetSystemVar(variable.CharacterSetConnection, test.chs)
require.NoError(t, err)
str, err := evalBuiltinFunc(f, chunk.Row{})

err = ctx.GetSessionVars().SetSystemVar(variable.CollationConnection, test.chs)
require.NoError(t, err)
var h []byte
if test.crypt != nil {
h, _ = hex.DecodeString(test.crypt.(string))
} else {
h = nil
}
f, err := newFunctionForTest(ctx, ast.Encode, primitiveValsToConstants(ctx, []interface{}{h, test.password})...)
require.NoError(t, err)
require.Equal(t, types.NewDatum(test.origin), str)
d, err := f.Eval(chunk.Row{})
require.NoError(t, err)
if test.origin != nil {
result, err := charset.NewEncoding(test.chs).EncodeString(test.origin.(string))
require.NoError(t, err)
require.Equal(t, types.NewCollationStringDatum(result, test.chs, 1), d)
} else {
result := types.NewDatum(test.origin)
require.Equal(t, result.GetBytes(), d.GetBytes())
}
}
testNullInput(t, ctx, ast.Encode)
}
Expand Down
36 changes: 30 additions & 6 deletions expression/builtin_encryption_vec.go
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,9 @@ func (b *builtinDecodeSig) vecEvalString(input *chunk.Chunk, result *chunk.Colum
if err := b.args[0].VecEvalString(b.ctx, input, buf); err != nil {
return err
}
bufTp := b.args[0].GetType()
bufEnc := charset.NewEncoding(bufTp.Charset)

buf1, err1 := b.bufAllocator.get()
if err1 != nil {
return err1
Expand All @@ -220,15 +223,24 @@ func (b *builtinDecodeSig) vecEvalString(input *chunk.Chunk, result *chunk.Colum
if err := b.args[1].VecEvalString(b.ctx, input, buf1); err != nil {
return err
}
buf1Tp := b.args[1].GetType()
buf1Enc := charset.NewEncoding(buf1Tp.Charset)
result.ReserveString(n)
var encodedBuf []byte
for i := 0; i < n; i++ {
if buf.IsNull(i) || buf1.IsNull(i) {
result.AppendNull()
continue
}
dataStr := buf.GetString(i)
passwordStr := buf1.GetString(i)
decodeStr, err := encrypt.SQLDecode(dataStr, passwordStr)
dataStr, err := bufEnc.Encode(encodedBuf, buf.GetBytes(i))
if err != nil {
return err
}
passwordStr, err := buf1Enc.Encode(encodedBuf, buf1.GetBytes(i))
if err != nil {
return err
}
decodeStr, err := encrypt.SQLDecodeBuf(dataStr, passwordStr)
if err != nil {
return err
}
Expand All @@ -255,19 +267,31 @@ func (b *builtinEncodeSig) vecEvalString(input *chunk.Chunk, result *chunk.Colum
if err1 != nil {
return err1
}
bufTp := b.args[0].GetType()
bufEnc := charset.NewEncoding(bufTp.Charset)
defer b.bufAllocator.put(buf1)
if err := b.args[1].VecEvalString(b.ctx, input, buf1); err != nil {
return err
}
buf1Tp := b.args[1].GetType()
buf1Enc := charset.NewEncoding(buf1Tp.Charset)
result.ReserveString(n)
var encodedBuf []byte
for i := 0; i < n; i++ {
if buf.IsNull(i) || buf1.IsNull(i) {
result.AppendNull()
continue
}
decodeStr := buf.GetString(i)
passwordStr := buf1.GetString(i)
dataStr, err := encrypt.SQLEncode(decodeStr, passwordStr)

decodeStr, err := bufEnc.Encode(encodedBuf, buf.GetBytes(i))
hawkingrei marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return err
}
passwordStr, err := buf1Enc.Encode(encodedBuf, buf1.GetBytes(i))
if err != nil {
return err
}
dataStr, err := encrypt.SQLEncodeBuf(decodeStr, passwordStr)
if err != nil {
return err
}
Expand Down
20 changes: 20 additions & 0 deletions util/encrypt/crypt.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,16 @@ func SQLDecode(str string, password string) (string, error) {
return string(strByte), nil
}

// SQLDecodeBuf Function to handle the decode() function
func SQLDecodeBuf(strByte []byte, passwdByte []byte) (string, error) {
var sc sqlCrypt

sc.init(passwdByte, len(passwdByte))
sc.decode(strByte, len(strByte))

return string(strByte), nil
}

// SQLEncode Function to handle the encode() function
func SQLEncode(cryptStr string, password string) (string, error) {
var sc sqlCrypt
Expand All @@ -132,3 +142,13 @@ func SQLEncode(cryptStr string, password string) (string, error) {

return string(cryptStrByte), nil
}

// SQLEncodeBuf Function to handle the encode() function
func SQLEncodeBuf(cryptStrByte []byte, passwdByte []byte) (string, error) {
var sc sqlCrypt

sc.init(passwdByte, len(passwdByte))
sc.encode(cryptStrByte, len(cryptStrByte))

return string(cryptStrByte), nil
}