From a2dc3eb571c5d3f88b09ae2683d6e660e994565f Mon Sep 17 00:00:00 2001 From: yibin hu Date: Tue, 21 Dec 2021 17:20:26 +0800 Subject: [PATCH 01/26] Fix greatest/least issue when handling duration type --- expression/builtin_compare.go | 2 +- expression/builtin_compare_test.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index 609ee26dc09e1..10e48826cf00e 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -432,7 +432,7 @@ func resolveType4Extremum(args []Expression) (_ types.EvalType, cmpStringAsDatet } } - if !types.IsTypeTemporal(aggType.Tp) && temporalItem != nil { + if !types.IsTypeTemporal(aggType.Tp) && temporalItem != nil && types.IsTemporalWithDate(aggType.Tp) { aggType.Tp = temporalItem.Tp cmpStringAsDatetime = true } diff --git a/expression/builtin_compare_test.go b/expression/builtin_compare_test.go index edc4b7e7ad056..cd204d25a2459 100644 --- a/expression/builtin_compare_test.go +++ b/expression/builtin_compare_test.go @@ -351,6 +351,10 @@ func TestGreatestLeastFunc(t *testing.T) { []interface{}{905969664.0, 4556, "1990-06-16 17:22:56.005534"}, "905969664", "1990-06-16 17:22:56.005534", false, false, }, + { + []interface{}{105969664.0, 120000, types.Duration{Duration: 20*time.Hour + 0*time.Minute + 0*time.Second}}, + "20:00:00", "105969664", false, false, + }, } { f0, err := newFunctionForTest(ctx, ast.Greatest, primitiveValsToConstants(ctx, test.args)...) require.NoError(t, err) From 29185d6609b06f6c257379c125ac0ab0deba53eb Mon Sep 17 00:00:00 2001 From: yibin87 Date: Thu, 23 Dec 2021 15:04:49 +0800 Subject: [PATCH 02/26] Fix some greatest/least function incorrect behavior when mixed types --- expression/builtin_compare.go | 64 ++++++++++++++++++++++++++-------- expression/integration_test.go | 23 ++++++++++++ go.mod | 2 +- go.sum | 4 +-- 4 files changed, 76 insertions(+), 17 deletions(-) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index 10e48826cf00e..a18748dfb52e5 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -417,7 +417,9 @@ func ResolveType4Between(args [3]Expression) types.EvalType { } // resolveType4Extremum gets compare type for GREATEST and LEAST and BETWEEN (mainly for datetime). -func resolveType4Extremum(args []Expression) (_ types.EvalType, cmpStringAsDatetime bool) { +// cmpStringMode: 0 for cmp string directly; 1 for cmp string as date 'yyyy-mm-dd'; 2 for cmp string +// as datetime 'yyyy-mm-dd hh:mm:ss' +func resolveType4Extremum(args []Expression) (_ types.EvalType, cmpStringMode int) { aggType := aggregateType(args) var temporalItem *types.FieldType @@ -432,13 +434,17 @@ func resolveType4Extremum(args []Expression) (_ types.EvalType, cmpStringAsDatet } } - if !types.IsTypeTemporal(aggType.Tp) && temporalItem != nil && types.IsTemporalWithDate(aggType.Tp) { + if !types.IsTypeTemporal(aggType.Tp) && temporalItem != nil && types.IsTemporalWithDate(temporalItem.Tp) { aggType.Tp = temporalItem.Tp - cmpStringAsDatetime = true + if aggType.Tp == mysql.TypeDate { + cmpStringMode = 1 + } else { + cmpStringMode = 2 + } } // TODO: String charset, collation checking are needed. } - return aggType.EvalType(), cmpStringAsDatetime + return aggType.EvalType(), cmpStringMode } // unsupportedJSONComparison reports warnings while there is a JSON type in least/greatest function's arguments @@ -460,8 +466,8 @@ func (c *greatestFunctionClass) getFunction(ctx sessionctx.Context, args []Expre if err = c.verifyArgs(args); err != nil { return nil, err } - tp, cmpStringAsDatetime := resolveType4Extremum(args) - if cmpStringAsDatetime { + tp, cmpStringMode := resolveType4Extremum(args) + if cmpStringMode > 0 { // Args are temporal and string mixed, we cast all args as string and parse it to temporal mannualy to compare. tp = types.ETString } else if tp == types.ETJson { @@ -495,8 +501,11 @@ func (c *greatestFunctionClass) getFunction(ctx sessionctx.Context, args []Expre sig = &builtinGreatestDecimalSig{bf} sig.setPbCode(tipb.ScalarFuncSig_GreatestDecimal) case types.ETString: - if cmpStringAsDatetime { - sig = &builtinGreatestCmpStringAsTimeSig{bf} + if cmpStringMode == 1 { + sig = &builtinGreatestCmpStringAsTimeSig{bf, true} + sig.setPbCode(tipb.ScalarFuncSig_GreatestCmpStringAsDate) + } else if cmpStringMode == 2 { + sig = &builtinGreatestCmpStringAsTimeSig{bf, false} sig.setPbCode(tipb.ScalarFuncSig_GreatestCmpStringAsTime) } else { sig = &builtinGreatestStringSig{bf} @@ -648,11 +657,13 @@ func (b *builtinGreatestStringSig) evalString(row chunk.Row) (max string, isNull type builtinGreatestCmpStringAsTimeSig struct { baseBuiltinFunc + cmpAsDate bool } func (b *builtinGreatestCmpStringAsTimeSig) Clone() builtinFunc { newSig := &builtinGreatestCmpStringAsTimeSig{} newSig.cloneFrom(&b.baseBuiltinFunc) + newSig.cmpAsDate = b.cmpAsDate return newSig } @@ -665,7 +676,12 @@ func (b *builtinGreatestCmpStringAsTimeSig) evalString(row chunk.Row) (strRes st if isNull || err != nil { return "", true, err } - t, err := types.ParseDatetime(sc, v) + var t types.Time + if b.cmpAsDate { + t, err = types.ParseDate(sc, v) + } else { + t, err = types.ParseDatetime(sc, v) + } if err != nil { if err = handleInvalidTimeError(b.ctx, err); err != nil { return v, true, err @@ -701,6 +717,11 @@ func (b *builtinGreatestTimeSig) evalTime(row chunk.Row) (res types.Time, isNull res = v } } + // Convert ETType Time value to MySQL actual type, distinguish date and datetime + sc := b.ctx.GetSessionVars().StmtCtx + if res, err = res.Convert(sc, b.tp.Tp); err != nil { + return types.ZeroTime, true, handleInvalidTimeError(b.ctx, err) + } return res, false, nil } @@ -735,8 +756,8 @@ func (c *leastFunctionClass) getFunction(ctx sessionctx.Context, args []Expressi if err = c.verifyArgs(args); err != nil { return nil, err } - tp, cmpStringAsDatetime := resolveType4Extremum(args) - if cmpStringAsDatetime { + tp, cmpStringMode := resolveType4Extremum(args) + if cmpStringMode > 0 { // Args are temporal and string mixed, we cast all args as string and parse it to temporal mannualy to compare. tp = types.ETString } else if tp == types.ETJson { @@ -770,8 +791,11 @@ func (c *leastFunctionClass) getFunction(ctx sessionctx.Context, args []Expressi sig = &builtinLeastDecimalSig{bf} sig.setPbCode(tipb.ScalarFuncSig_LeastDecimal) case types.ETString: - if cmpStringAsDatetime { - sig = &builtinLeastCmpStringAsTimeSig{bf} + if cmpStringMode == 1 { + sig = &builtinLeastCmpStringAsTimeSig{bf, true} + sig.setPbCode(tipb.ScalarFuncSig_LeastCmpStringAsDate) + } else if cmpStringMode == 2 { + sig = &builtinLeastCmpStringAsTimeSig{bf, false} sig.setPbCode(tipb.ScalarFuncSig_LeastCmpStringAsTime) } else { sig = &builtinLeastStringSig{bf} @@ -910,11 +934,13 @@ func (b *builtinLeastStringSig) evalString(row chunk.Row) (min string, isNull bo type builtinLeastCmpStringAsTimeSig struct { baseBuiltinFunc + cmpAsDate bool } func (b *builtinLeastCmpStringAsTimeSig) Clone() builtinFunc { newSig := &builtinLeastCmpStringAsTimeSig{} newSig.cloneFrom(&b.baseBuiltinFunc) + newSig.cmpAsDate = b.cmpAsDate return newSig } @@ -927,7 +953,12 @@ func (b *builtinLeastCmpStringAsTimeSig) evalString(row chunk.Row) (strRes strin if isNull || err != nil { return "", true, err } - t, err := types.ParseDatetime(sc, v) + var t types.Time + if b.cmpAsDate { + t, err = types.ParseDate(sc, v) + } else { + t, err = types.ParseDatetime(sc, v) + } if err != nil { if err = handleInvalidTimeError(b.ctx, err); err != nil { return v, true, err @@ -963,6 +994,11 @@ func (b *builtinLeastTimeSig) evalTime(row chunk.Row) (res types.Time, isNull bo res = v } } + // Convert ETType Time value to MySQL actual type, distinguish date and datetime + sc := b.ctx.GetSessionVars().StmtCtx + if res, err = res.Convert(sc, b.tp.Tp); err != nil { + return types.ZeroTime, true, handleInvalidTimeError(b.ctx, err) + } return res, false, nil } diff --git a/expression/integration_test.go b/expression/integration_test.go index 95dc3157507ad..1bf3e9553a722 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -6927,3 +6927,26 @@ func TestIssue30174(t *testing.T) { tk.MustQuery("select * from t2 where c3 in (select c2 from t1);").Check(testkit.Rows()) tk.MustQuery("select * from t2 where c2 in (select c2 from t1);").Check(testkit.Rows()) } + +func TestIssue30264(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + // compare Time/Int/Int as string type, return string type + tk.MustQuery("select greatest(time '21:00', year(date'20220101'), 23);").Check(testkit.Rows("23")) + // compare Time/Date/Int as string type, return string type + tk.MustQuery("select greatest(time '21:00', date'891001', 120000);").Check(testkit.Rows("21:00:00")) + // compare Time/Date/Int as string type, return string type + tk.MustQuery("select greatest(time '20:00', date'101001', 120101);").Check(testkit.Rows("20:00:00")) + // compare Date/String/Int as Date type, return string type + tk.MustQuery("select greatest(date'101001', '19990329', 120101);").Check(testkit.Rows("2012-01-01")) + // compare Time/Date as DateTime type, return DateTime type + tk.MustQuery("select greatest(time '20:00', date'691231');").Check(testkit.Rows("2069-12-31 00:00:00")) + //Original 30264 Issue: + tk.MustQuery("select greatest(time '20:00:00', 120000);").Check(testkit.Rows("20:00:00")) + tk.MustQuery("select greatest(date '2005-05-05', 20010101, 20040404, 20030303);").Check(testkit.Rows("2005-05-05")) + tk.MustQuery("select greatest(date '1995-05-05', 19910101, 20050505, 19930303);").Check(testkit.Rows("2005-05-05")) + +} diff --git a/go.mod b/go.mod index f11eab1cf0456..65647b70d8904 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ require ( github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d github.com/pingcap/tidb-tools v5.2.2-0.20211019062242-37a8bef2fa17+incompatible github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e - github.com/pingcap/tipb v0.0.0-20211201080053-bd104bb270ba + github.com/pingcap/tipb v0.0.0-20211223061136-6a2bd37d5c31 github.com/prometheus/client_golang v1.5.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.9.1 diff --git a/go.sum b/go.sum index 2a4c23d9d70bc..12b5e997bb2e9 100644 --- a/go.sum +++ b/go.sum @@ -598,8 +598,8 @@ github.com/pingcap/tidb-dashboard v0.0.0-20211008050453-a25c25809529/go.mod h1:O github.com/pingcap/tidb-dashboard v0.0.0-20211107164327-80363dfbe884/go.mod h1:OCXbZTBTIMRcIt0jFsuCakZP+goYRv6IjawKbwLS2TQ= github.com/pingcap/tidb-tools v5.2.2-0.20211019062242-37a8bef2fa17+incompatible h1:c7+izmker91NkjkZ6FgTlmD4k1A5FLOAq+li6Ki2/GY= github.com/pingcap/tidb-tools v5.2.2-0.20211019062242-37a8bef2fa17+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tipb v0.0.0-20211201080053-bd104bb270ba h1:Tt5W/maVBUbG+wxg2nfc88Cqj/HiWYb0TJQ2Rfi0UOQ= -github.com/pingcap/tipb v0.0.0-20211201080053-bd104bb270ba/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= +github.com/pingcap/tipb v0.0.0-20211223061136-6a2bd37d5c31 h1:USv7VGrZxqJQ+gH0Bfuk5fBtWz6v6X8dGfGz4goQ0oU= +github.com/pingcap/tipb v0.0.0-20211223061136-6a2bd37d5c31/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= From 4a29734bf54adcaea5582f047c91eff2c271c39f Mon Sep 17 00:00:00 2001 From: yibin87 Date: Tue, 21 Dec 2021 17:20:26 +0800 Subject: [PATCH 03/26] Fix greatest/least issue when handling duration type --- expression/builtin_compare.go | 2 +- expression/builtin_compare_test.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index 609ee26dc09e1..10e48826cf00e 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -432,7 +432,7 @@ func resolveType4Extremum(args []Expression) (_ types.EvalType, cmpStringAsDatet } } - if !types.IsTypeTemporal(aggType.Tp) && temporalItem != nil { + if !types.IsTypeTemporal(aggType.Tp) && temporalItem != nil && types.IsTemporalWithDate(aggType.Tp) { aggType.Tp = temporalItem.Tp cmpStringAsDatetime = true } diff --git a/expression/builtin_compare_test.go b/expression/builtin_compare_test.go index edc4b7e7ad056..cd204d25a2459 100644 --- a/expression/builtin_compare_test.go +++ b/expression/builtin_compare_test.go @@ -351,6 +351,10 @@ func TestGreatestLeastFunc(t *testing.T) { []interface{}{905969664.0, 4556, "1990-06-16 17:22:56.005534"}, "905969664", "1990-06-16 17:22:56.005534", false, false, }, + { + []interface{}{105969664.0, 120000, types.Duration{Duration: 20*time.Hour + 0*time.Minute + 0*time.Second}}, + "20:00:00", "105969664", false, false, + }, } { f0, err := newFunctionForTest(ctx, ast.Greatest, primitiveValsToConstants(ctx, test.args)...) require.NoError(t, err) From 545dec226fceab46d69410066da026163e5ed533 Mon Sep 17 00:00:00 2001 From: yibin hu Date: Mon, 27 Dec 2021 12:47:50 +0800 Subject: [PATCH 04/26] Separate GreatestTime and GreatestDate, LeastTime and LeastDate, and add integration tests --- expression/builtin_compare.go | 84 ++++++++++++++++++++++++++-------- expression/distsql_builtin.go | 16 ++++++- expression/integration_test.go | 11 ++++- go.mod | 2 +- go.sum | 4 +- 5 files changed, 92 insertions(+), 25 deletions(-) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index a18748dfb52e5..2707f71cf15c0 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -419,7 +419,8 @@ func ResolveType4Between(args [3]Expression) types.EvalType { // resolveType4Extremum gets compare type for GREATEST and LEAST and BETWEEN (mainly for datetime). // cmpStringMode: 0 for cmp string directly; 1 for cmp string as date 'yyyy-mm-dd'; 2 for cmp string // as datetime 'yyyy-mm-dd hh:mm:ss' -func resolveType4Extremum(args []Expression) (_ types.EvalType, cmpStringMode int) { +// fieldTimeType: the return type, 0 for non-temporary time; 1 for date type; 2 for datetime type +func resolveType4Extremum(args []Expression) (_ types.EvalType, fieldTimeType byte, cmpStringMode int) { aggType := aggregateType(args) var temporalItem *types.FieldType @@ -435,8 +436,7 @@ func resolveType4Extremum(args []Expression) (_ types.EvalType, cmpStringMode in } if !types.IsTypeTemporal(aggType.Tp) && temporalItem != nil && types.IsTemporalWithDate(temporalItem.Tp) { - aggType.Tp = temporalItem.Tp - if aggType.Tp == mysql.TypeDate { + if temporalItem.Tp == mysql.TypeDate { cmpStringMode = 1 } else { cmpStringMode = 2 @@ -444,7 +444,13 @@ func resolveType4Extremum(args []Expression) (_ types.EvalType, cmpStringMode in } // TODO: String charset, collation checking are needed. } - return aggType.EvalType(), cmpStringMode + var timeType byte + if aggType.Tp == mysql.TypeDate { + timeType = 1 + } else if aggType.Tp == mysql.TypeDatetime || aggType.Tp == mysql.TypeTimestamp { + timeType = 2 + } + return aggType.EvalType(), timeType, cmpStringMode } // unsupportedJSONComparison reports warnings while there is a JSON type in least/greatest function's arguments @@ -466,23 +472,24 @@ func (c *greatestFunctionClass) getFunction(ctx sessionctx.Context, args []Expre if err = c.verifyArgs(args); err != nil { return nil, err } - tp, cmpStringMode := resolveType4Extremum(args) + tp, fieldTimeType, cmpStringMode := resolveType4Extremum(args) + argTp := tp if cmpStringMode > 0 { // Args are temporal and string mixed, we cast all args as string and parse it to temporal mannualy to compare. - tp = types.ETString + argTp = types.ETString } else if tp == types.ETJson { unsupportedJSONComparison(ctx, args) - tp = types.ETString + argTp = types.ETString } argTps := make([]types.EvalType, len(args)) for i := range args { - argTps[i] = tp + argTps[i] = argTp } bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, tp, argTps...) if err != nil { return nil, err } - switch tp { + switch argTp { case types.ETInt: // adjust unsigned flag greastInitUnsignedFlag := false @@ -515,8 +522,13 @@ func (c *greatestFunctionClass) getFunction(ctx sessionctx.Context, args []Expre sig = &builtinGreatestDurationSig{bf} sig.setPbCode(tipb.ScalarFuncSig_GreatestDuration) case types.ETDatetime, types.ETTimestamp: - sig = &builtinGreatestTimeSig{bf} - sig.setPbCode(tipb.ScalarFuncSig_GreatestTime) + if fieldTimeType == 1 { + sig = &builtinGreatestTimeSig{bf, true} + sig.setPbCode(tipb.ScalarFuncSig_GreatestDate) + } else { + sig = &builtinGreatestTimeSig{bf, false} + sig.setPbCode(tipb.ScalarFuncSig_GreatestTime) + } } sig.getRetTp().Flen, sig.getRetTp().Decimal = fixFlenAndDecimalForGreatestAndLeast(args) return sig, nil @@ -679,8 +691,14 @@ func (b *builtinGreatestCmpStringAsTimeSig) evalString(row chunk.Row) (strRes st var t types.Time if b.cmpAsDate { t, err = types.ParseDate(sc, v) + if err == nil { + t, err = t.Convert(sc, mysql.TypeDate) + } } else { t, err = types.ParseDatetime(sc, v) + if err == nil { + t, err = t.Convert(sc, mysql.TypeDatetime) + } } if err != nil { if err = handleInvalidTimeError(b.ctx, err); err != nil { @@ -699,11 +717,13 @@ func (b *builtinGreatestCmpStringAsTimeSig) evalString(row chunk.Row) (strRes st type builtinGreatestTimeSig struct { baseBuiltinFunc + cmpAsDate bool } func (b *builtinGreatestTimeSig) Clone() builtinFunc { newSig := &builtinGreatestTimeSig{} newSig.cloneFrom(&b.baseBuiltinFunc) + newSig.cmpAsDate = b.cmpAsDate return newSig } @@ -719,7 +739,13 @@ func (b *builtinGreatestTimeSig) evalTime(row chunk.Row) (res types.Time, isNull } // Convert ETType Time value to MySQL actual type, distinguish date and datetime sc := b.ctx.GetSessionVars().StmtCtx - if res, err = res.Convert(sc, b.tp.Tp); err != nil { + var resTimeTp byte + if b.cmpAsDate { + resTimeTp = mysql.TypeDate + } else { + resTimeTp = mysql.TypeDatetime + } + if res, err = res.Convert(sc, resTimeTp); err != nil { return types.ZeroTime, true, handleInvalidTimeError(b.ctx, err) } return res, false, nil @@ -756,17 +782,18 @@ func (c *leastFunctionClass) getFunction(ctx sessionctx.Context, args []Expressi if err = c.verifyArgs(args); err != nil { return nil, err } - tp, cmpStringMode := resolveType4Extremum(args) + tp, fieldTimeType, cmpStringMode := resolveType4Extremum(args) + argTp := tp if cmpStringMode > 0 { // Args are temporal and string mixed, we cast all args as string and parse it to temporal mannualy to compare. - tp = types.ETString + argTp = types.ETString } else if tp == types.ETJson { unsupportedJSONComparison(ctx, args) - tp = types.ETString + argTp = types.ETString } argTps := make([]types.EvalType, len(args)) for i := range args { - argTps[i] = tp + argTps[i] = argTp } bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, tp, argTps...) if err != nil { @@ -805,8 +832,13 @@ func (c *leastFunctionClass) getFunction(ctx sessionctx.Context, args []Expressi sig = &builtinLeastDurationSig{bf} sig.setPbCode(tipb.ScalarFuncSig_LeastDuration) case types.ETDatetime, types.ETTimestamp: - sig = &builtinLeastTimeSig{bf} - sig.setPbCode(tipb.ScalarFuncSig_LeastTime) + if fieldTimeType == 1 { + sig = &builtinLeastTimeSig{bf, true} + sig.setPbCode(tipb.ScalarFuncSig_LeastDate) + } else { + sig = &builtinLeastTimeSig{bf, false} + sig.setPbCode(tipb.ScalarFuncSig_LeastTime) + } } sig.getRetTp().Flen, sig.getRetTp().Decimal = fixFlenAndDecimalForGreatestAndLeast(args) return sig, nil @@ -956,8 +988,14 @@ func (b *builtinLeastCmpStringAsTimeSig) evalString(row chunk.Row) (strRes strin var t types.Time if b.cmpAsDate { t, err = types.ParseDate(sc, v) + if err == nil { + t, err = t.Convert(sc, mysql.TypeDate) + } } else { t, err = types.ParseDatetime(sc, v) + if err == nil { + t, err = t.Convert(sc, mysql.TypeDatetime) + } } if err != nil { if err = handleInvalidTimeError(b.ctx, err); err != nil { @@ -976,11 +1014,13 @@ func (b *builtinLeastCmpStringAsTimeSig) evalString(row chunk.Row) (strRes strin type builtinLeastTimeSig struct { baseBuiltinFunc + cmpAsDate bool } func (b *builtinLeastTimeSig) Clone() builtinFunc { newSig := &builtinLeastTimeSig{} newSig.cloneFrom(&b.baseBuiltinFunc) + newSig.cmpAsDate = b.cmpAsDate return newSig } @@ -996,7 +1036,13 @@ func (b *builtinLeastTimeSig) evalTime(row chunk.Row) (res types.Time, isNull bo } // Convert ETType Time value to MySQL actual type, distinguish date and datetime sc := b.ctx.GetSessionVars().StmtCtx - if res, err = res.Convert(sc, b.tp.Tp); err != nil { + var resTimeTp byte + if b.cmpAsDate { + resTimeTp = mysql.TypeDate + } else { + resTimeTp = mysql.TypeDatetime + } + if res, err = res.Convert(sc, resTimeTp); err != nil { return types.ZeroTime, true, handleInvalidTimeError(b.ctx, err) } return res, false, nil diff --git a/expression/distsql_builtin.go b/expression/distsql_builtin.go index 47dd46f87b39f..d1f054a9cb1dd 100644 --- a/expression/distsql_builtin.go +++ b/expression/distsql_builtin.go @@ -222,7 +222,13 @@ func getSignatureByPB(ctx sessionctx.Context, sigCode tipb.ScalarFuncSig, tp *ti case tipb.ScalarFuncSig_GreatestString: f = &builtinGreatestStringSig{base} case tipb.ScalarFuncSig_GreatestTime: - f = &builtinGreatestTimeSig{base} + f = &builtinGreatestTimeSig{base, false} + case tipb.ScalarFuncSig_GreatestDate: + f = &builtinGreatestTimeSig{base, true} + case tipb.ScalarFuncSig_GreatestCmpStringAsTime: + f = &builtinGreatestCmpStringAsTimeSig{base, false} + case tipb.ScalarFuncSig_GreatestCmpStringAsDate: + f = &builtinGreatestCmpStringAsTimeSig{base, true} case tipb.ScalarFuncSig_LeastInt: f = &builtinLeastIntSig{base} case tipb.ScalarFuncSig_LeastReal: @@ -232,7 +238,13 @@ func getSignatureByPB(ctx sessionctx.Context, sigCode tipb.ScalarFuncSig, tp *ti case tipb.ScalarFuncSig_LeastString: f = &builtinLeastStringSig{base} case tipb.ScalarFuncSig_LeastTime: - f = &builtinLeastTimeSig{base} + f = &builtinLeastTimeSig{base, false} + case tipb.ScalarFuncSig_LeastDate: + f = &builtinLeastTimeSig{base, true} + case tipb.ScalarFuncSig_LeastCmpStringAsTime: + f = &builtinLeastCmpStringAsTimeSig{base, false} + case tipb.ScalarFuncSig_LeastCmpStringAsDate: + f = &builtinLeastCmpStringAsTimeSig{base, true} case tipb.ScalarFuncSig_IntervalInt: f = &builtinIntervalIntSig{base, false} // Since interval function won't be pushed down to TiKV, therefore it doesn't matter what value we give to hasNullable case tipb.ScalarFuncSig_IntervalReal: diff --git a/expression/integration_test.go b/expression/integration_test.go index 1bf3e9553a722..90a7bb7897e00 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -6944,9 +6944,18 @@ func TestIssue30264(t *testing.T) { tk.MustQuery("select greatest(date'101001', '19990329', 120101);").Check(testkit.Rows("2012-01-01")) // compare Time/Date as DateTime type, return DateTime type tk.MustQuery("select greatest(time '20:00', date'691231');").Check(testkit.Rows("2069-12-31 00:00:00")) + // compare Date/Date as Date type, return Date type + tk.MustQuery("select greatest(date '120301', date'691231');").Check(testkit.Rows("2069-12-31")) + // compare Time/Time as Time type, return Time type + tk.MustQuery("select greatest(time '203001', time '2230');").Check(testkit.Rows("20:30:01")) + // compare DateTime/DateTime as DateTime type, return DateTime type + tk.MustQuery("select greatest(timestamp '2021-01-31 00:00:01', timestamp '2021-12-31 12:00:00');").Check(testkit.Rows("2021-12-31 12:00:00")) + // compare Time/DateTime as DateTime type, return DateTime type + tk.MustQuery("select greatest(time '00:00:01', timestamp '2069-12-31 12:00:00');").Check(testkit.Rows("2069-12-31 12:00:00")) + // compare Date/DateTime as DateTime type, return DateTime type + tk.MustQuery("select greatest(date '21000101', timestamp '2069-12-31 12:00:00');").Check(testkit.Rows("2100-01-01 00:00:00")) //Original 30264 Issue: tk.MustQuery("select greatest(time '20:00:00', 120000);").Check(testkit.Rows("20:00:00")) tk.MustQuery("select greatest(date '2005-05-05', 20010101, 20040404, 20030303);").Check(testkit.Rows("2005-05-05")) tk.MustQuery("select greatest(date '1995-05-05', 19910101, 20050505, 19930303);").Check(testkit.Rows("2005-05-05")) - } diff --git a/go.mod b/go.mod index 65647b70d8904..732470d532123 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ require ( github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d github.com/pingcap/tidb-tools v5.2.2-0.20211019062242-37a8bef2fa17+incompatible github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e - github.com/pingcap/tipb v0.0.0-20211223061136-6a2bd37d5c31 + github.com/pingcap/tipb v0.0.0-20211227035912-f7f1de26ca3d github.com/prometheus/client_golang v1.5.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.9.1 diff --git a/go.sum b/go.sum index 12b5e997bb2e9..1fadfbec4b138 100644 --- a/go.sum +++ b/go.sum @@ -598,8 +598,8 @@ github.com/pingcap/tidb-dashboard v0.0.0-20211008050453-a25c25809529/go.mod h1:O github.com/pingcap/tidb-dashboard v0.0.0-20211107164327-80363dfbe884/go.mod h1:OCXbZTBTIMRcIt0jFsuCakZP+goYRv6IjawKbwLS2TQ= github.com/pingcap/tidb-tools v5.2.2-0.20211019062242-37a8bef2fa17+incompatible h1:c7+izmker91NkjkZ6FgTlmD4k1A5FLOAq+li6Ki2/GY= github.com/pingcap/tidb-tools v5.2.2-0.20211019062242-37a8bef2fa17+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tipb v0.0.0-20211223061136-6a2bd37d5c31 h1:USv7VGrZxqJQ+gH0Bfuk5fBtWz6v6X8dGfGz4goQ0oU= -github.com/pingcap/tipb v0.0.0-20211223061136-6a2bd37d5c31/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= +github.com/pingcap/tipb v0.0.0-20211227035912-f7f1de26ca3d h1:X+2esV9AhKCq1axej0un3Eld09SL7feH9rQ7O1gGDcg= +github.com/pingcap/tipb v0.0.0-20211227035912-f7f1de26ca3d/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= From fb273c785ad5cc8d77482dab2e0fde9881ac43fb Mon Sep 17 00:00:00 2001 From: yibin hu Date: Tue, 28 Dec 2021 17:22:22 +0800 Subject: [PATCH 05/26] Add support for vector greatest/least functions --- expression/builtin_compare.go | 45 +++++++++++++---------- expression/builtin_compare_vec.go | 61 ++++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 24 deletions(-) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index 2707f71cf15c0..806570a2c0d3b 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -415,14 +415,21 @@ func ResolveType4Between(args [3]Expression) types.EvalType { return cmpTp } - +type CmpStringMode uint8 +const ( + CmpStringDirectly CmpStringMode = iota + CmpStringAsDate // 'yyyy-mm-dd' + CmpStringAsDatetime // 'yyyy-mm-dd hh:mm:ss' +) +type GLRetTimeType uint8 //Greatest/Least return time type +const ( + GLRetNoneTemporal GLRetTimeType = iota + GLRetDate // 'yyyy-mm-dd' + GLRetDatetime // 'yyyy-mm-dd hh:mm:ss' +) // resolveType4Extremum gets compare type for GREATEST and LEAST and BETWEEN (mainly for datetime). -// cmpStringMode: 0 for cmp string directly; 1 for cmp string as date 'yyyy-mm-dd'; 2 for cmp string -// as datetime 'yyyy-mm-dd hh:mm:ss' -// fieldTimeType: the return type, 0 for non-temporary time; 1 for date type; 2 for datetime type -func resolveType4Extremum(args []Expression) (_ types.EvalType, fieldTimeType byte, cmpStringMode int) { +func resolveType4Extremum(args []Expression) (_ types.EvalType, fieldTimeType GLRetTimeType, cmpStringMode CmpStringMode) { aggType := aggregateType(args) - var temporalItem *types.FieldType if aggType.EvalType().IsStringKind() { for i := range args { @@ -437,18 +444,18 @@ func resolveType4Extremum(args []Expression) (_ types.EvalType, fieldTimeType by if !types.IsTypeTemporal(aggType.Tp) && temporalItem != nil && types.IsTemporalWithDate(temporalItem.Tp) { if temporalItem.Tp == mysql.TypeDate { - cmpStringMode = 1 + cmpStringMode = CmpStringAsDate } else { - cmpStringMode = 2 + cmpStringMode = CmpStringAsDatetime } } // TODO: String charset, collation checking are needed. } - var timeType byte + var timeType GLRetTimeType if aggType.Tp == mysql.TypeDate { - timeType = 1 + timeType = GLRetDate } else if aggType.Tp == mysql.TypeDatetime || aggType.Tp == mysql.TypeTimestamp { - timeType = 2 + timeType = GLRetDatetime } return aggType.EvalType(), timeType, cmpStringMode } @@ -474,7 +481,7 @@ func (c *greatestFunctionClass) getFunction(ctx sessionctx.Context, args []Expre } tp, fieldTimeType, cmpStringMode := resolveType4Extremum(args) argTp := tp - if cmpStringMode > 0 { + if cmpStringMode != CmpStringDirectly { // Args are temporal and string mixed, we cast all args as string and parse it to temporal mannualy to compare. argTp = types.ETString } else if tp == types.ETJson { @@ -508,10 +515,10 @@ func (c *greatestFunctionClass) getFunction(ctx sessionctx.Context, args []Expre sig = &builtinGreatestDecimalSig{bf} sig.setPbCode(tipb.ScalarFuncSig_GreatestDecimal) case types.ETString: - if cmpStringMode == 1 { + if cmpStringMode == CmpStringAsDate { sig = &builtinGreatestCmpStringAsTimeSig{bf, true} sig.setPbCode(tipb.ScalarFuncSig_GreatestCmpStringAsDate) - } else if cmpStringMode == 2 { + } else if cmpStringMode == CmpStringAsDatetime { sig = &builtinGreatestCmpStringAsTimeSig{bf, false} sig.setPbCode(tipb.ScalarFuncSig_GreatestCmpStringAsTime) } else { @@ -522,7 +529,7 @@ func (c *greatestFunctionClass) getFunction(ctx sessionctx.Context, args []Expre sig = &builtinGreatestDurationSig{bf} sig.setPbCode(tipb.ScalarFuncSig_GreatestDuration) case types.ETDatetime, types.ETTimestamp: - if fieldTimeType == 1 { + if fieldTimeType == GLRetDate { sig = &builtinGreatestTimeSig{bf, true} sig.setPbCode(tipb.ScalarFuncSig_GreatestDate) } else { @@ -784,7 +791,7 @@ func (c *leastFunctionClass) getFunction(ctx sessionctx.Context, args []Expressi } tp, fieldTimeType, cmpStringMode := resolveType4Extremum(args) argTp := tp - if cmpStringMode > 0 { + if cmpStringMode != CmpStringDirectly { // Args are temporal and string mixed, we cast all args as string and parse it to temporal mannualy to compare. argTp = types.ETString } else if tp == types.ETJson { @@ -818,10 +825,10 @@ func (c *leastFunctionClass) getFunction(ctx sessionctx.Context, args []Expressi sig = &builtinLeastDecimalSig{bf} sig.setPbCode(tipb.ScalarFuncSig_LeastDecimal) case types.ETString: - if cmpStringMode == 1 { + if cmpStringMode == CmpStringAsDate { sig = &builtinLeastCmpStringAsTimeSig{bf, true} sig.setPbCode(tipb.ScalarFuncSig_LeastCmpStringAsDate) - } else if cmpStringMode == 2 { + } else if cmpStringMode == CmpStringAsDatetime { sig = &builtinLeastCmpStringAsTimeSig{bf, false} sig.setPbCode(tipb.ScalarFuncSig_LeastCmpStringAsTime) } else { @@ -832,7 +839,7 @@ func (c *leastFunctionClass) getFunction(ctx sessionctx.Context, args []Expressi sig = &builtinLeastDurationSig{bf} sig.setPbCode(tipb.ScalarFuncSig_LeastDuration) case types.ETDatetime, types.ETTimestamp: - if fieldTimeType == 1 { + if fieldTimeType == GLRetDate { sig = &builtinLeastTimeSig{bf, true} sig.setPbCode(tipb.ScalarFuncSig_LeastDate) } else { diff --git a/expression/builtin_compare_vec.go b/expression/builtin_compare_vec.go index e3bf42864fd94..28097f4bfc50b 100644 --- a/expression/builtin_compare_vec.go +++ b/expression/builtin_compare_vec.go @@ -652,14 +652,26 @@ func (b *builtinGreatestCmpStringAsTimeSig) vecEvalString(input *chunk.Chunk, re // NOTE: can't use Column.GetString because it returns an unsafe string, copy the row instead. argTimeStr := string(result.GetBytes(i)) + var t types.Time + var err error + if b.cmpAsDate { + t, err := types.ParseDate(sc, argTimeStr) + if err == nil { + t, err = t.Convert(sc, mysql.TypeDate) + } + } else { + t, err := types.ParseDatetime(sc, argTimeStr) + if err == nil { + t, err = t.Convert(sc, mysql.TypeDatetime) + } + } - argTime, err := types.ParseDatetime(sc, argTimeStr) if err != nil { if err = handleInvalidTimeError(b.ctx, err); err != nil { return err } } else { - argTimeStr = argTime.String() + argTimeStr = t.String() } if j == 0 || strings.Compare(argTimeStr, dstStrings[i]) > 0 { dstStrings[i] = argTimeStr @@ -737,14 +749,25 @@ func (b *builtinLeastCmpStringAsTimeSig) vecEvalString(input *chunk.Chunk, resul // NOTE: can't use Column.GetString because it returns an unsafe string, copy the row instead. argTimeStr := string(result.GetBytes(i)) - - argTime, err := types.ParseDatetime(sc, argTimeStr) + var t types.Time + var err error + if b.cmpAsDate { + t, err := types.ParseDate(sc, argTimeStr) + if err == nil { + t, err = t.Convert(sc, mysql.TypeDate) + } + } else { + t, err := types.ParseDatetime(sc, argTimeStr) + if err == nil { + t, err = t.Convert(sc, mysql.TypeDatetime) + } + } if err != nil { if err = handleInvalidTimeError(b.ctx, err); err != nil { return err } } else { - argTimeStr = argTime.String() + argTimeStr = t.String() } if j == 0 || strings.Compare(argTimeStr, dstStrings[i]) < 0 { dstStrings[i] = argTimeStr @@ -845,6 +868,20 @@ func (b *builtinGreatestTimeSig) vecEvalTime(input *chunk.Chunk, result *chunk.C } } } + sc := b.ctx.GetSessionVars().StmtCtx + var resTimeTp byte + if b.cmpAsDate { + resTimeTp = mysql.TypeDate + } else { + resTimeTp = mysql.TypeDatetime + } + for rowIdx := 0; rowIdx < n; rowIdx++ { + resTimes := result.Times() + resTimes[rowIdx], err = resTimes[rowIdx].Convert(sc, resTimeTp) + if err != nil { + return err + } + } return nil } @@ -877,6 +914,20 @@ func (b *builtinLeastTimeSig) vecEvalTime(input *chunk.Chunk, result *chunk.Colu } } } + sc := b.ctx.GetSessionVars().StmtCtx + var resTimeTp byte + if b.cmpAsDate { + resTimeTp = mysql.TypeDate + } else { + resTimeTp = mysql.TypeDatetime + } + for rowIdx := 0; rowIdx < n; rowIdx++ { + resTimes := result.Times() + resTimes[rowIdx], err = resTimes[rowIdx].Convert(sc, resTimeTp) + if err != nil { + return err + } + } return nil } From e1574b1f6e80ca4f1c35fb2b0d4be72bc620f6ea Mon Sep 17 00:00:00 2001 From: yibin hu Date: Tue, 28 Dec 2021 18:40:31 +0800 Subject: [PATCH 06/26] make format --- expression/builtin_compare.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index 806570a2c0d3b..83c0fe7a1cbb1 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -415,18 +415,22 @@ func ResolveType4Between(args [3]Expression) types.EvalType { return cmpTp } + type CmpStringMode uint8 + const ( - CmpStringDirectly CmpStringMode = iota - CmpStringAsDate // 'yyyy-mm-dd' - CmpStringAsDatetime // 'yyyy-mm-dd hh:mm:ss' + CmpStringDirectly CmpStringMode = iota + CmpStringAsDate // 'yyyy-mm-dd' + CmpStringAsDatetime // 'yyyy-mm-dd hh:mm:ss' ) -type GLRetTimeType uint8 //Greatest/Least return time type + +type GLRetTimeType uint8 //Greatest/Least return time type const ( GLRetNoneTemporal GLRetTimeType = iota - GLRetDate // 'yyyy-mm-dd' - GLRetDatetime // 'yyyy-mm-dd hh:mm:ss' + GLRetDate // 'yyyy-mm-dd' + GLRetDatetime // 'yyyy-mm-dd hh:mm:ss' ) + // resolveType4Extremum gets compare type for GREATEST and LEAST and BETWEEN (mainly for datetime). func resolveType4Extremum(args []Expression) (_ types.EvalType, fieldTimeType GLRetTimeType, cmpStringMode CmpStringMode) { aggType := aggregateType(args) @@ -451,7 +455,7 @@ func resolveType4Extremum(args []Expression) (_ types.EvalType, fieldTimeType GL } // TODO: String charset, collation checking are needed. } - var timeType GLRetTimeType + var timeType = GLRetNoneTemporal if aggType.Tp == mysql.TypeDate { timeType = GLRetDate } else if aggType.Tp == mysql.TypeDatetime || aggType.Tp == mysql.TypeTimestamp { From 5c4aef30d18fc7aae25e2e458f984bc06460b7dc Mon Sep 17 00:00:00 2001 From: yibin hu Date: Tue, 28 Dec 2021 19:03:18 +0800 Subject: [PATCH 07/26] Format enum definitions --- expression/builtin_compare.go | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index 83c0fe7a1cbb1..d15df07b2f73b 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -416,15 +416,18 @@ func ResolveType4Between(args [3]Expression) types.EvalType { return cmpTp } -type CmpStringMode uint8 +//Greatest/Least interal string comparison mode +type GLCmpStringMode uint8 const ( - CmpStringDirectly CmpStringMode = iota - CmpStringAsDate // 'yyyy-mm-dd' - CmpStringAsDatetime // 'yyyy-mm-dd hh:mm:ss' + GLCmpStringDirectly GLCmpStringMode = iota + GLCmpStringAsDate // 'yyyy-mm-dd' + GLCmpStringAsDatetime // 'yyyy-mm-dd hh:mm:ss' ) -type GLRetTimeType uint8 //Greatest/Least return time type +//Greatest/Least return time type +type GLRetTimeType uint8 + const ( GLRetNoneTemporal GLRetTimeType = iota GLRetDate // 'yyyy-mm-dd' @@ -432,7 +435,7 @@ const ( ) // resolveType4Extremum gets compare type for GREATEST and LEAST and BETWEEN (mainly for datetime). -func resolveType4Extremum(args []Expression) (_ types.EvalType, fieldTimeType GLRetTimeType, cmpStringMode CmpStringMode) { +func resolveType4Extremum(args []Expression) (_ types.EvalType, fieldTimeType GLRetTimeType, cmpStringMode GLCmpStringMode) { aggType := aggregateType(args) var temporalItem *types.FieldType if aggType.EvalType().IsStringKind() { @@ -448,9 +451,9 @@ func resolveType4Extremum(args []Expression) (_ types.EvalType, fieldTimeType GL if !types.IsTypeTemporal(aggType.Tp) && temporalItem != nil && types.IsTemporalWithDate(temporalItem.Tp) { if temporalItem.Tp == mysql.TypeDate { - cmpStringMode = CmpStringAsDate + cmpStringMode = GLCmpStringAsDate } else { - cmpStringMode = CmpStringAsDatetime + cmpStringMode = GLCmpStringAsDatetime } } // TODO: String charset, collation checking are needed. @@ -485,7 +488,7 @@ func (c *greatestFunctionClass) getFunction(ctx sessionctx.Context, args []Expre } tp, fieldTimeType, cmpStringMode := resolveType4Extremum(args) argTp := tp - if cmpStringMode != CmpStringDirectly { + if cmpStringMode != GLCmpStringDirectly { // Args are temporal and string mixed, we cast all args as string and parse it to temporal mannualy to compare. argTp = types.ETString } else if tp == types.ETJson { @@ -519,10 +522,10 @@ func (c *greatestFunctionClass) getFunction(ctx sessionctx.Context, args []Expre sig = &builtinGreatestDecimalSig{bf} sig.setPbCode(tipb.ScalarFuncSig_GreatestDecimal) case types.ETString: - if cmpStringMode == CmpStringAsDate { + if cmpStringMode == GLCmpStringAsDate { sig = &builtinGreatestCmpStringAsTimeSig{bf, true} sig.setPbCode(tipb.ScalarFuncSig_GreatestCmpStringAsDate) - } else if cmpStringMode == CmpStringAsDatetime { + } else if cmpStringMode == GLCmpStringAsDatetime { sig = &builtinGreatestCmpStringAsTimeSig{bf, false} sig.setPbCode(tipb.ScalarFuncSig_GreatestCmpStringAsTime) } else { @@ -795,7 +798,7 @@ func (c *leastFunctionClass) getFunction(ctx sessionctx.Context, args []Expressi } tp, fieldTimeType, cmpStringMode := resolveType4Extremum(args) argTp := tp - if cmpStringMode != CmpStringDirectly { + if cmpStringMode != GLCmpStringDirectly { // Args are temporal and string mixed, we cast all args as string and parse it to temporal mannualy to compare. argTp = types.ETString } else if tp == types.ETJson { @@ -829,10 +832,10 @@ func (c *leastFunctionClass) getFunction(ctx sessionctx.Context, args []Expressi sig = &builtinLeastDecimalSig{bf} sig.setPbCode(tipb.ScalarFuncSig_LeastDecimal) case types.ETString: - if cmpStringMode == CmpStringAsDate { + if cmpStringMode == GLCmpStringAsDate { sig = &builtinLeastCmpStringAsTimeSig{bf, true} sig.setPbCode(tipb.ScalarFuncSig_LeastCmpStringAsDate) - } else if cmpStringMode == CmpStringAsDatetime { + } else if cmpStringMode == GLCmpStringAsDatetime { sig = &builtinLeastCmpStringAsTimeSig{bf, false} sig.setPbCode(tipb.ScalarFuncSig_LeastCmpStringAsTime) } else { From 6ad60a3cac3178ab5cfc0f2e4bcc6687ec655319 Mon Sep 17 00:00:00 2001 From: yibin hu Date: Tue, 28 Dec 2021 19:19:05 +0800 Subject: [PATCH 08/26] Format enum definitions --- expression/builtin_compare.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index d15df07b2f73b..f4cce00d1f6fb 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -416,7 +416,7 @@ func ResolveType4Between(args [3]Expression) types.EvalType { return cmpTp } -//Greatest/Least interal string comparison mode +// GLCmpStringMode represents Greatest/Least interal string comparison mode type GLCmpStringMode uint8 const ( @@ -425,7 +425,7 @@ const ( GLCmpStringAsDatetime // 'yyyy-mm-dd hh:mm:ss' ) -//Greatest/Least return time type +// GLRetTimeType represents Greatest/Least return time type type GLRetTimeType uint8 const ( From 3003a03a25fc0e5d7c208c994f8513880bd662af Mon Sep 17 00:00:00 2001 From: yibin hu Date: Tue, 28 Dec 2021 19:27:54 +0800 Subject: [PATCH 09/26] Format enum definitions --- expression/builtin_compare.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index f4cce00d1f6fb..bafa28350f8df 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -420,18 +420,24 @@ func ResolveType4Between(args [3]Expression) types.EvalType { type GLCmpStringMode uint8 const ( + // Greatest/Least function compares string directly GLCmpStringDirectly GLCmpStringMode = iota - GLCmpStringAsDate // 'yyyy-mm-dd' - GLCmpStringAsDatetime // 'yyyy-mm-dd hh:mm:ss' + // Greatest/Least function compares string as 'yyyy-mm-dd' format + GLCmpStringAsDate + // Greatest/Least function compares string as 'yyyy-mm-dd hh:mm:ss' format + GLCmpStringAsDatetime ) // GLRetTimeType represents Greatest/Least return time type type GLRetTimeType uint8 const ( + // Greatest/Least function returns non temporal time GLRetNoneTemporal GLRetTimeType = iota - GLRetDate // 'yyyy-mm-dd' - GLRetDatetime // 'yyyy-mm-dd hh:mm:ss' + // Greatest/Least function returns date type, 'yyyy-mm-dd' + GLRetDate + // Greatest/Least function returns datetime type, 'yyyy-mm-dd hh:mm:ss' + GLRetDatetime ) // resolveType4Extremum gets compare type for GREATEST and LEAST and BETWEEN (mainly for datetime). From 9e571ff980a23319308392906970265baece2c98 Mon Sep 17 00:00:00 2001 From: yibin hu Date: Wed, 29 Dec 2021 09:23:27 +0800 Subject: [PATCH 10/26] Format change --- expression/builtin_compare.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index bafa28350f8df..d0a774a4b1f26 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -421,7 +421,7 @@ type GLCmpStringMode uint8 const ( // Greatest/Least function compares string directly - GLCmpStringDirectly GLCmpStringMode = iota + GLCmpStringDirectly GLCmpStringMode = iota // Greatest/Least function compares string as 'yyyy-mm-dd' format GLCmpStringAsDate // Greatest/Least function compares string as 'yyyy-mm-dd hh:mm:ss' format From 18244f0317dd563a426eddbd38fc500d3c77aee2 Mon Sep 17 00:00:00 2001 From: yibin hu Date: Wed, 29 Dec 2021 09:45:35 +0800 Subject: [PATCH 11/26] Format change --- expression/builtin_compare.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index d0a774a4b1f26..376243ea85b8e 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -420,11 +420,11 @@ func ResolveType4Between(args [3]Expression) types.EvalType { type GLCmpStringMode uint8 const ( - // Greatest/Least function compares string directly + // GLCmpStringDirectly: Greatest/Least function compares string directly GLCmpStringDirectly GLCmpStringMode = iota - // Greatest/Least function compares string as 'yyyy-mm-dd' format + // GLCmpStringAsDate: Greatest/Least function compares string as 'yyyy-mm-dd' format GLCmpStringAsDate - // Greatest/Least function compares string as 'yyyy-mm-dd hh:mm:ss' format + // GLCmpStringAsDatetime: Greatest/Least function compares string as 'yyyy-mm-dd hh:mm:ss' format GLCmpStringAsDatetime ) @@ -432,11 +432,11 @@ const ( type GLRetTimeType uint8 const ( - // Greatest/Least function returns non temporal time + // GLRetNoneTemporal: Greatest/Least function returns non temporal time GLRetNoneTemporal GLRetTimeType = iota - // Greatest/Least function returns date type, 'yyyy-mm-dd' + // GLRetDate: Greatest/Least function returns date type, 'yyyy-mm-dd' GLRetDate - // Greatest/Least function returns datetime type, 'yyyy-mm-dd hh:mm:ss' + // GLRetDatetime: Greatest/Least function returns datetime type, 'yyyy-mm-dd hh:mm:ss' GLRetDatetime ) From f1fc0329b5ee3b56da716ef17fb2f694c6496d8c Mon Sep 17 00:00:00 2001 From: yibin hu Date: Wed, 29 Dec 2021 10:29:18 +0800 Subject: [PATCH 12/26] Format change --- expression/builtin_compare.go | 12 ++++++------ expression/integration_test.go | 13 +++++++++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index 376243ea85b8e..10dd9324a2b88 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -420,11 +420,11 @@ func ResolveType4Between(args [3]Expression) types.EvalType { type GLCmpStringMode uint8 const ( - // GLCmpStringDirectly: Greatest/Least function compares string directly + // GLCmpStringDirectly Greatest/Least function compares string directly GLCmpStringDirectly GLCmpStringMode = iota - // GLCmpStringAsDate: Greatest/Least function compares string as 'yyyy-mm-dd' format + // GLCmpStringAsDate Greatest/Least function compares string as 'yyyy-mm-dd' format GLCmpStringAsDate - // GLCmpStringAsDatetime: Greatest/Least function compares string as 'yyyy-mm-dd hh:mm:ss' format + // GLCmpStringAsDatetime Greatest/Least function compares string as 'yyyy-mm-dd hh:mm:ss' format GLCmpStringAsDatetime ) @@ -432,11 +432,11 @@ const ( type GLRetTimeType uint8 const ( - // GLRetNoneTemporal: Greatest/Least function returns non temporal time + // GLRetNoneTemporal Greatest/Least function returns non temporal time GLRetNoneTemporal GLRetTimeType = iota - // GLRetDate: Greatest/Least function returns date type, 'yyyy-mm-dd' + // GLRetDate Greatest/Least function returns date type, 'yyyy-mm-dd' GLRetDate - // GLRetDatetime: Greatest/Least function returns datetime type, 'yyyy-mm-dd hh:mm:ss' + // GLRetDatetime Greatest/Least function returns datetime type, 'yyyy-mm-dd hh:mm:ss' GLRetDatetime ) diff --git a/expression/integration_test.go b/expression/integration_test.go index 90a7bb7897e00..15c26a3b1be6d 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -6959,3 +6959,16 @@ func TestIssue30264(t *testing.T) { tk.MustQuery("select greatest(date '2005-05-05', 20010101, 20040404, 20030303);").Check(testkit.Rows("2005-05-05")) tk.MustQuery("select greatest(date '1995-05-05', 19910101, 20050505, 19930303);").Check(testkit.Rows("2005-05-05")) } + +func TestIssue30174Vec(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1,t2;") + tk.MustExec("CREATE TABLE `t1` (\n `c1` enum('Alice','Bob','Charlie','David') NOT NULL,\n `c2` blob NOT NULL,\n PRIMARY KEY (`c2`(5)),\n UNIQUE KEY `idx_89` (`c1`)\n);") + tk.MustExec("insert into t1 values('Charlie','');") + tk.MustExec("insert into t2 values('Charlie','Charlie','Alice');") + tk.MustQuery("select * from t2 where c3 in (select c2 from t1);").Check(testkit.Rows()) + tk.MustQuery("select * from t2 where c2 in (select c2 from t1);").Check(testkit.Rows()) +} From 367ab6eade62554ffe1ceff3574c2aebdf49ecc1 Mon Sep 17 00:00:00 2001 From: yibin hu Date: Wed, 29 Dec 2021 20:15:37 +0800 Subject: [PATCH 13/26] Refact code and add vec test cases --- expression/builtin_compare.go | 85 +++++++++++++++---------------- expression/builtin_compare_vec.go | 53 +++---------------- expression/integration_test.go | 33 ++++++++---- 3 files changed, 69 insertions(+), 102 deletions(-) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index 10dd9324a2b88..cc81e611ac16d 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -15,6 +15,7 @@ package expression import ( + "github.com/pingcap/tidb/sessionctx/stmtctx" "math" "strings" @@ -708,24 +709,9 @@ func (b *builtinGreatestCmpStringAsTimeSig) evalString(row chunk.Row) (strRes st if isNull || err != nil { return "", true, err } - var t types.Time - if b.cmpAsDate { - t, err = types.ParseDate(sc, v) - if err == nil { - t, err = t.Convert(sc, mysql.TypeDate) - } - } else { - t, err = types.ParseDatetime(sc, v) - if err == nil { - t, err = t.Convert(sc, mysql.TypeDatetime) - } - } + v, err = doTimeConversionForGL(b.cmpAsDate, b.ctx, sc, v) if err != nil { - if err = handleInvalidTimeError(b.ctx, err); err != nil { - return v, true, err - } - } else { - v = t.String() + return v, true, err } // In MySQL, if the compare result is zero, than we will try to use the string comparison result if i == 0 || strings.Compare(v, strRes) > 0 { @@ -735,6 +721,30 @@ func (b *builtinGreatestCmpStringAsTimeSig) evalString(row chunk.Row) (strRes st return strRes, false, nil } +func doTimeConversionForGL(cmpAsDate bool, ctx sessionctx.Context, sc *stmtctx.StatementContext, strVal string) (string, error) { + var t types.Time + var err error + if cmpAsDate { + t, err = types.ParseDate(sc, strVal) + if err == nil { + t, err = t.Convert(sc, mysql.TypeDate) + } + } else { + t, err = types.ParseDatetime(sc, strVal) + if err == nil { + t, err = t.Convert(sc, mysql.TypeDatetime) + } + } + if err != nil { + if err = handleInvalidTimeError(ctx, err); err != nil { + return "", err + } + } else { + strVal = t.String() + } + return strVal, nil +} + type builtinGreatestTimeSig struct { baseBuiltinFunc cmpAsDate bool @@ -759,12 +769,7 @@ func (b *builtinGreatestTimeSig) evalTime(row chunk.Row) (res types.Time, isNull } // Convert ETType Time value to MySQL actual type, distinguish date and datetime sc := b.ctx.GetSessionVars().StmtCtx - var resTimeTp byte - if b.cmpAsDate { - resTimeTp = mysql.TypeDate - } else { - resTimeTp = mysql.TypeDatetime - } + resTimeTp := getAccurateTimeTypeForGLRet(b.cmpAsDate) if res, err = res.Convert(sc, resTimeTp); err != nil { return types.ZeroTime, true, handleInvalidTimeError(b.ctx, err) } @@ -1005,24 +1010,9 @@ func (b *builtinLeastCmpStringAsTimeSig) evalString(row chunk.Row) (strRes strin if isNull || err != nil { return "", true, err } - var t types.Time - if b.cmpAsDate { - t, err = types.ParseDate(sc, v) - if err == nil { - t, err = t.Convert(sc, mysql.TypeDate) - } - } else { - t, err = types.ParseDatetime(sc, v) - if err == nil { - t, err = t.Convert(sc, mysql.TypeDatetime) - } - } + v, err = doTimeConversionForGL(b.cmpAsDate, b.ctx, sc, v) if err != nil { - if err = handleInvalidTimeError(b.ctx, err); err != nil { - return v, true, err - } - } else { - v = t.String() + return v, true, err } if i == 0 || strings.Compare(v, strRes) < 0 { strRes = v @@ -1056,16 +1046,21 @@ func (b *builtinLeastTimeSig) evalTime(row chunk.Row) (res types.Time, isNull bo } // Convert ETType Time value to MySQL actual type, distinguish date and datetime sc := b.ctx.GetSessionVars().StmtCtx + resTimeTp := getAccurateTimeTypeForGLRet(b.cmpAsDate) + if res, err = res.Convert(sc, resTimeTp); err != nil { + return types.ZeroTime, true, handleInvalidTimeError(b.ctx, err) + } + return res, false, nil +} + +func getAccurateTimeTypeForGLRet(cmpAsDate bool) byte { var resTimeTp byte - if b.cmpAsDate { + if cmpAsDate { resTimeTp = mysql.TypeDate } else { resTimeTp = mysql.TypeDatetime } - if res, err = res.Convert(sc, resTimeTp); err != nil { - return types.ZeroTime, true, handleInvalidTimeError(b.ctx, err) - } - return res, false, nil + return resTimeTp } type builtinLeastDurationSig struct { diff --git a/expression/builtin_compare_vec.go b/expression/builtin_compare_vec.go index 28097f4bfc50b..35663bb8418d6 100644 --- a/expression/builtin_compare_vec.go +++ b/expression/builtin_compare_vec.go @@ -652,26 +652,10 @@ func (b *builtinGreatestCmpStringAsTimeSig) vecEvalString(input *chunk.Chunk, re // NOTE: can't use Column.GetString because it returns an unsafe string, copy the row instead. argTimeStr := string(result.GetBytes(i)) - var t types.Time var err error - if b.cmpAsDate { - t, err := types.ParseDate(sc, argTimeStr) - if err == nil { - t, err = t.Convert(sc, mysql.TypeDate) - } - } else { - t, err := types.ParseDatetime(sc, argTimeStr) - if err == nil { - t, err = t.Convert(sc, mysql.TypeDatetime) - } - } - + argTimeStr, err = doTimeConversionForGL(b.cmpAsDate, b.ctx, sc, argTimeStr) if err != nil { - if err = handleInvalidTimeError(b.ctx, err); err != nil { - return err - } - } else { - argTimeStr = t.String() + return err } if j == 0 || strings.Compare(argTimeStr, dstStrings[i]) > 0 { dstStrings[i] = argTimeStr @@ -749,25 +733,10 @@ func (b *builtinLeastCmpStringAsTimeSig) vecEvalString(input *chunk.Chunk, resul // NOTE: can't use Column.GetString because it returns an unsafe string, copy the row instead. argTimeStr := string(result.GetBytes(i)) - var t types.Time var err error - if b.cmpAsDate { - t, err := types.ParseDate(sc, argTimeStr) - if err == nil { - t, err = t.Convert(sc, mysql.TypeDate) - } - } else { - t, err := types.ParseDatetime(sc, argTimeStr) - if err == nil { - t, err = t.Convert(sc, mysql.TypeDatetime) - } - } + argTimeStr, err = doTimeConversionForGL(b.cmpAsDate, b.ctx, sc, argTimeStr) if err != nil { - if err = handleInvalidTimeError(b.ctx, err); err != nil { - return err - } - } else { - argTimeStr = t.String() + return err } if j == 0 || strings.Compare(argTimeStr, dstStrings[i]) < 0 { dstStrings[i] = argTimeStr @@ -869,12 +838,7 @@ func (b *builtinGreatestTimeSig) vecEvalTime(input *chunk.Chunk, result *chunk.C } } sc := b.ctx.GetSessionVars().StmtCtx - var resTimeTp byte - if b.cmpAsDate { - resTimeTp = mysql.TypeDate - } else { - resTimeTp = mysql.TypeDatetime - } + resTimeTp := getAccurateTimeTypeForGLRet(b.cmpAsDate) for rowIdx := 0; rowIdx < n; rowIdx++ { resTimes := result.Times() resTimes[rowIdx], err = resTimes[rowIdx].Convert(sc, resTimeTp) @@ -915,12 +879,7 @@ func (b *builtinLeastTimeSig) vecEvalTime(input *chunk.Chunk, result *chunk.Colu } } sc := b.ctx.GetSessionVars().StmtCtx - var resTimeTp byte - if b.cmpAsDate { - resTimeTp = mysql.TypeDate - } else { - resTimeTp = mysql.TypeDatetime - } + resTimeTp := getAccurateTimeTypeForGLRet(b.cmpAsDate) for rowIdx := 0; rowIdx < n; rowIdx++ { resTimes := result.Times() resTimes[rowIdx], err = resTimes[rowIdx].Convert(sc, resTimeTp) diff --git a/expression/integration_test.go b/expression/integration_test.go index 15c26a3b1be6d..0827793a0fbef 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -6958,17 +6958,30 @@ func TestIssue30264(t *testing.T) { tk.MustQuery("select greatest(time '20:00:00', 120000);").Check(testkit.Rows("20:00:00")) tk.MustQuery("select greatest(date '2005-05-05', 20010101, 20040404, 20030303);").Check(testkit.Rows("2005-05-05")) tk.MustQuery("select greatest(date '1995-05-05', 19910101, 20050505, 19930303);").Check(testkit.Rows("2005-05-05")) -} -func TestIssue30174Vec(t *testing.T) { - store, clean := testkit.CreateMockStore(t) - defer clean() - tk := testkit.NewTestKit(t, store) tk.MustExec("use test") tk.MustExec("drop table if exists t1,t2;") - tk.MustExec("CREATE TABLE `t1` (\n `c1` enum('Alice','Bob','Charlie','David') NOT NULL,\n `c2` blob NOT NULL,\n PRIMARY KEY (`c2`(5)),\n UNIQUE KEY `idx_89` (`c1`)\n);") - tk.MustExec("insert into t1 values('Charlie','');") - tk.MustExec("insert into t2 values('Charlie','Charlie','Alice');") - tk.MustQuery("select * from t2 where c3 in (select c2 from t1);").Check(testkit.Rows()) - tk.MustQuery("select * from t2 where c2 in (select c2 from t1);").Check(testkit.Rows()) + tk.MustExec("CREATE TABLE `t1` (a datetime, b date, c time)") + tk.MustExec("insert into t1 values(timestamp'2021-01-31 00:00:01', '2069-12-31', '20:00:01');") + tk.MustExec("set tidb_enable_vectorized_expression = on;") + // compare Time/Int/Int as string type, return string type + tk.MustQuery("select greatest(c, year(date'20220101'), 23) from t1;").Check(testkit.Rows("23")) + // compare Time/Date/Int as string type, return string type + tk.MustQuery("select greatest(c, date'891001', 120000) from t1;").Check(testkit.Rows("20:00:01")) + // compare Time/Date/Int as string type, return string type + tk.MustQuery("select greatest(c, date'101001', 120101) from t1;").Check(testkit.Rows("20:00:01")) + // compare Date/String/Int as Date type, return string type + tk.MustQuery("select greatest(b, '19990329', 120101) from t1;").Check(testkit.Rows("2069-12-31")) + // compare Time/Date as DateTime type, return DateTime type + tk.MustQuery("select greatest(time '20:00', b) from t1;").Check(testkit.Rows("2069-12-31 00:00:00")) + // compare Date/Date as Date type, return Date type + tk.MustQuery("select greatest(date '120301', b) from t1;").Check(testkit.Rows("2069-12-31")) + // compare Time/Time as Time type, return Time type + tk.MustQuery("select greatest(c, time '2230') from t1;").Check(testkit.Rows("20:00:01")) + // compare DateTime/DateTime as DateTime type, return DateTime type + tk.MustQuery("select greatest(a, timestamp '2021-12-31 12:00:00') from t1;").Check(testkit.Rows("2021-12-31 12:00:00")) + // compare Time/DateTime as DateTime type, return DateTime type + tk.MustQuery("select greatest(c, timestamp '2069-12-31 12:00:00') from t1;").Check(testkit.Rows("2069-12-31 12:00:00")) + // compare Date/DateTime as DateTime type, return DateTime type + tk.MustQuery("select greatest(date '21000101', a) from t1;").Check(testkit.Rows("2100-01-01 00:00:00")) } From b74586719eaafcf2b7cfc8128dad8bde9cd93544 Mon Sep 17 00:00:00 2001 From: yibin hu Date: Wed, 29 Dec 2021 20:38:21 +0800 Subject: [PATCH 14/26] Fmt changes --- expression/builtin_compare.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index cc81e611ac16d..931cf174d150e 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -15,20 +15,21 @@ package expression import ( - "github.com/pingcap/tidb/sessionctx/stmtctx" "math" "strings" + "github.com/pingcap/tipb/go-tipb" + "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/parser/opcode" "github.com/pingcap/tidb/parser/terror" "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/types/json" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/collate" - "github.com/pingcap/tipb/go-tipb" ) var ( From 1e11ca0b1c70c08d88d60b0ff417161648412604 Mon Sep 17 00:00:00 2001 From: yibin hu Date: Wed, 29 Dec 2021 20:58:24 +0800 Subject: [PATCH 15/26] Modify comment to trigger unit test --- expression/builtin_compare.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index 931cf174d150e..c4af4f7bdadb2 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -422,7 +422,7 @@ func ResolveType4Between(args [3]Expression) types.EvalType { type GLCmpStringMode uint8 const ( - // GLCmpStringDirectly Greatest/Least function compares string directly + // GLCmpStringDirectly Greatest and Least function compares string directly GLCmpStringDirectly GLCmpStringMode = iota // GLCmpStringAsDate Greatest/Least function compares string as 'yyyy-mm-dd' format GLCmpStringAsDate From d4505c37ea9798543f7bd4545ef4f6f13a27b8ac Mon Sep 17 00:00:00 2001 From: yibin hu Date: Thu, 30 Dec 2021 09:41:47 +0800 Subject: [PATCH 16/26] Adjust import sequence --- expression/builtin_compare.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index c4af4f7bdadb2..5dc6ef29e58d3 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -18,8 +18,6 @@ import ( "math" "strings" - "github.com/pingcap/tipb/go-tipb" - "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/parser/opcode" @@ -30,6 +28,7 @@ import ( "github.com/pingcap/tidb/types/json" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/collate" + "github.com/pingcap/tipb/go-tipb" ) var ( From 88fd4d6e3c80c81c6a3680a4fcaebddfa70c1248 Mon Sep 17 00:00:00 2001 From: yibin hu Date: Thu, 30 Dec 2021 10:49:09 +0800 Subject: [PATCH 17/26] Fix mysql test json case --- expression/builtin_compare.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index 5dc6ef29e58d3..b0e1670d5cdbc 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -501,6 +501,7 @@ func (c *greatestFunctionClass) getFunction(ctx sessionctx.Context, args []Expre } else if tp == types.ETJson { unsupportedJSONComparison(ctx, args) argTp = types.ETString + tp = types.ETString } argTps := make([]types.EvalType, len(args)) for i := range args { @@ -815,6 +816,7 @@ func (c *leastFunctionClass) getFunction(ctx sessionctx.Context, args []Expressi } else if tp == types.ETJson { unsupportedJSONComparison(ctx, args) argTp = types.ETString + tp = types.ETString } argTps := make([]types.EvalType, len(args)) for i := range args { From 356c372ad74ea0c63d2395aabda2694b00783fbf Mon Sep 17 00:00:00 2001 From: yibin hu Date: Thu, 30 Dec 2021 11:00:35 +0800 Subject: [PATCH 18/26] Add JSON tests --- expression/integration_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/expression/integration_test.go b/expression/integration_test.go index 0827793a0fbef..9477c5524d0c6 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -6954,6 +6954,8 @@ func TestIssue30264(t *testing.T) { tk.MustQuery("select greatest(time '00:00:01', timestamp '2069-12-31 12:00:00');").Check(testkit.Rows("2069-12-31 12:00:00")) // compare Date/DateTime as DateTime type, return DateTime type tk.MustQuery("select greatest(date '21000101', timestamp '2069-12-31 12:00:00');").Check(testkit.Rows("2100-01-01 00:00:00")) + // compare JSON/JSON, return JSON type + tk.MustQuery("select greatest(cast('1' as JSON), cast('2' as JSON));").Check(testkit.Rows("2")) //Original 30264 Issue: tk.MustQuery("select greatest(time '20:00:00', 120000);").Check(testkit.Rows("20:00:00")) tk.MustQuery("select greatest(date '2005-05-05', 20010101, 20040404, 20030303);").Check(testkit.Rows("2005-05-05")) @@ -6984,4 +6986,6 @@ func TestIssue30264(t *testing.T) { tk.MustQuery("select greatest(c, timestamp '2069-12-31 12:00:00') from t1;").Check(testkit.Rows("2069-12-31 12:00:00")) // compare Date/DateTime as DateTime type, return DateTime type tk.MustQuery("select greatest(date '21000101', a) from t1;").Check(testkit.Rows("2100-01-01 00:00:00")) + // compare JSON/JSON, return JSON type + tk.MustQuery("select greatest(cast(a as JSON), cast('3' as JSON)) from t1;").Check(testkit.Rows("3")) } From feb8f4788b1159bba2ebd7feaf5aa604842cee8f Mon Sep 17 00:00:00 2001 From: yibin Date: Thu, 6 Jan 2022 19:31:02 +0800 Subject: [PATCH 19/26] Fix wrap cast function that droped nullable information --- planner/core/rule_aggregation_elimination.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/planner/core/rule_aggregation_elimination.go b/planner/core/rule_aggregation_elimination.go index 9adca6936099d..f6831108711bb 100644 --- a/planner/core/rule_aggregation_elimination.go +++ b/planner/core/rule_aggregation_elimination.go @@ -208,6 +208,9 @@ func wrapCastFunction(ctx sessionctx.Context, arg expression.Expression, targetT if arg.GetType().Equal(targetTp) { return arg } + if !mysql.HasNotNullFlag(arg.GetType().Flag) { + targetTp.Flag &= ^mysql.NotNullFlag + } return expression.BuildCastFunction(ctx, arg, targetTp) } From 08f0dc0a2e8356f2182533ebd86451aeb3092f71 Mon Sep 17 00:00:00 2001 From: yibin Date: Fri, 7 Jan 2022 13:20:34 +0800 Subject: [PATCH 20/26] Add plan test --- planner/core/logical_plan_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index 0136545eff430..5497275b913df 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -110,6 +110,23 @@ func (s *testPlanSuite) TestPredicatePushDown(c *C) { } } +// Issue: 31399 +func (s *testPlanSuite) TestImplicitCastNotNullFlag(c *C) { + defer testleak.AfterTest(c)() + ctx := context.Background() + ca := "select count(*) from t3 group by a having bit_and(b) > 1;" + comment := Commentf("for %s", ca) + stmt, err := s.ParseOneStmt(ca, "", "") + c.Assert(err, IsNil, comment) + p, _, err := BuildLogicalPlanForTest(ctx, s.ctx, stmt, s.is) + c.Assert(err, IsNil) + p, err = logicalOptimize(context.TODO(), flagPredicatePushDown|flagJoinReOrder|flagPrunColumns|flagEliminateProjection, p.(LogicalPlan)) + c.Assert(err, IsNil) + // AggFuncs[0] is count; AggFuncs[1] is bit_and, args + castNullFlag := (p.(*LogicalProjection).children[0].(*LogicalSelection).children[0].(*LogicalAggregation).AggFuncs[1].Args[0].GetType().Flag) & mysql.NotNullFlag + c.Assert(castNullFlag, Equals, 0) +} + func (s *testPlanSuite) TestEliminateProjectionUnderUnion(c *C) { defer testleak.AfterTest(c)() ctx := context.Background() From 7e9bb6b3921000486fad55537c0ab6bb4b22d144 Mon Sep 17 00:00:00 2001 From: yibin Date: Fri, 7 Jan 2022 13:22:03 +0800 Subject: [PATCH 21/26] Refact --- planner/core/logical_plan_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index 5497275b913df..b666287a68aeb 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -123,8 +123,8 @@ func (s *testPlanSuite) TestImplicitCastNotNullFlag(c *C) { p, err = logicalOptimize(context.TODO(), flagPredicatePushDown|flagJoinReOrder|flagPrunColumns|flagEliminateProjection, p.(LogicalPlan)) c.Assert(err, IsNil) // AggFuncs[0] is count; AggFuncs[1] is bit_and, args - castNullFlag := (p.(*LogicalProjection).children[0].(*LogicalSelection).children[0].(*LogicalAggregation).AggFuncs[1].Args[0].GetType().Flag) & mysql.NotNullFlag - c.Assert(castNullFlag, Equals, 0) + castNotNullFlag := (p.(*LogicalProjection).children[0].(*LogicalSelection).children[0].(*LogicalAggregation).AggFuncs[1].Args[0].GetType().Flag) & mysql.NotNullFlag + c.Assert(castNotNullFlag, Equals, 0) } func (s *testPlanSuite) TestEliminateProjectionUnderUnion(c *C) { From ea8e16760e18aa5bd87b38e10b06a437e24cf254 Mon Sep 17 00:00:00 2001 From: yibin Date: Fri, 7 Jan 2022 13:29:48 +0800 Subject: [PATCH 22/26] Refact --- planner/core/logical_plan_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index b666287a68aeb..d14d77ea3b8b5 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -122,7 +122,7 @@ func (s *testPlanSuite) TestImplicitCastNotNullFlag(c *C) { c.Assert(err, IsNil) p, err = logicalOptimize(context.TODO(), flagPredicatePushDown|flagJoinReOrder|flagPrunColumns|flagEliminateProjection, p.(LogicalPlan)) c.Assert(err, IsNil) - // AggFuncs[0] is count; AggFuncs[1] is bit_and, args + // AggFuncs[0] is count; AggFuncs[1] is bit_and, args[0] is return type of the implicit cast castNotNullFlag := (p.(*LogicalProjection).children[0].(*LogicalSelection).children[0].(*LogicalAggregation).AggFuncs[1].Args[0].GetType().Flag) & mysql.NotNullFlag c.Assert(castNotNullFlag, Equals, 0) } From bb968ae69cb4f446e406d9e439062dce540833c8 Mon Sep 17 00:00:00 2001 From: yibin Date: Fri, 7 Jan 2022 17:09:26 +0800 Subject: [PATCH 23/26] Move nullable check to lower level implementations --- expression/builtin_cast.go | 5 +++++ planner/core/rule_aggregation_elimination.go | 3 --- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/expression/builtin_cast.go b/expression/builtin_cast.go index 6cbd7933745b9..54b4abf6d817c 100644 --- a/expression/builtin_cast.go +++ b/expression/builtin_cast.go @@ -1840,6 +1840,11 @@ func BuildCastCollationFunction(ctx sessionctx.Context, expr Expression, ec *Exp // BuildCastFunction builds a CAST ScalarFunction from the Expression. func BuildCastFunction(ctx sessionctx.Context, expr Expression, tp *types.FieldType) (res Expression) { + argType := expr.GetType() + // If source argument's nullable, then target type should be nullable + if !mysql.HasNotNullFlag(argType.Flag) { + tp.Flag &= ^mysql.NotNullFlag + } expr = TryPushCastIntoControlFunctionForHybridType(ctx, expr, tp) var fc functionClass switch tp.EvalType() { diff --git a/planner/core/rule_aggregation_elimination.go b/planner/core/rule_aggregation_elimination.go index f6831108711bb..9adca6936099d 100644 --- a/planner/core/rule_aggregation_elimination.go +++ b/planner/core/rule_aggregation_elimination.go @@ -208,9 +208,6 @@ func wrapCastFunction(ctx sessionctx.Context, arg expression.Expression, targetT if arg.GetType().Equal(targetTp) { return arg } - if !mysql.HasNotNullFlag(arg.GetType().Flag) { - targetTp.Flag &= ^mysql.NotNullFlag - } return expression.BuildCastFunction(ctx, arg, targetTp) } From cfde28a2d2b53a13663c56c98f3feeafa03333d4 Mon Sep 17 00:00:00 2001 From: yibin Date: Mon, 24 Jan 2022 10:45:08 +0800 Subject: [PATCH 24/26] Fix a uint/int 0 not equal issue in remote unit test --- planner/core/logical_plan_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index d14d77ea3b8b5..aec01a73e8442 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -124,7 +124,8 @@ func (s *testPlanSuite) TestImplicitCastNotNullFlag(c *C) { c.Assert(err, IsNil) // AggFuncs[0] is count; AggFuncs[1] is bit_and, args[0] is return type of the implicit cast castNotNullFlag := (p.(*LogicalProjection).children[0].(*LogicalSelection).children[0].(*LogicalAggregation).AggFuncs[1].Args[0].GetType().Flag) & mysql.NotNullFlag - c.Assert(castNotNullFlag, Equals, 0) + var nullableFlag uint = 0 + c.Assert(castNotNullFlag, Equals, nullableFlag) } func (s *testPlanSuite) TestEliminateProjectionUnderUnion(c *C) { From d40ad26500993f50efe4de6c948f03538e9bf573 Mon Sep 17 00:00:00 2001 From: yibin Date: Wed, 23 Feb 2022 17:08:29 +0800 Subject: [PATCH 25/26] enable DayName/MonthName function push-down for tiflash --- expression/expression.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/expression/expression.go b/expression/expression.go index 0fc49b1adbc49..14e29c6a795d7 100644 --- a/expression/expression.go +++ b/expression/expression.go @@ -1039,7 +1039,7 @@ func scalarExprSupportedByFlash(function *ScalarFunction) bool { ast.Plus, ast.Minus, ast.Div, ast.Mul, ast.Abs, ast.Mod, ast.If, ast.Ifnull, ast.Case, ast.Concat, ast.ConcatWS, - ast.Date, ast.Year, ast.Month, ast.Day, ast.Quarter, + ast.Date, ast.Year, ast.Month, ast.Day, ast.Quarter, ast.DayName, ast.MonthName, ast.DateDiff, ast.TimestampDiff, ast.DateFormat, ast.FromUnixTime, ast.Sqrt, ast.Log, ast.Log2, ast.Log10, ast.Ln, ast.Exp, ast.Pow, ast.Sign, From 91aa3a6c7f261049d651f0e915a640a1a418cccf Mon Sep 17 00:00:00 2001 From: yibin Date: Thu, 24 Feb 2022 13:53:43 +0800 Subject: [PATCH 26/26] Add push-down unit test for dayname/monthname --- expression/expr_to_pb_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/expression/expr_to_pb_test.go b/expression/expr_to_pb_test.go index e1829b53338a3..0363701692a7c 100644 --- a/expression/expr_to_pb_test.go +++ b/expression/expr_to_pb_test.go @@ -1015,6 +1015,16 @@ func TestExprPushDownToFlash(t *testing.T) { require.NoError(t, err) exprs = append(exprs, function) + // DayName: supported + function, err = NewFunction(mock.NewContext(), ast.DayName, types.NewFieldType(mysql.TypeString), datetimeColumn) + require.NoError(t, err) + exprs = append(exprs, function) + + // MonthName: supported + function, err = NewFunction(mock.NewContext(), ast.MonthName, types.NewFieldType(mysql.TypeString), datetimeColumn) + require.NoError(t, err) + exprs = append(exprs, function) + pushed, remained = PushDownExprs(sc, exprs, client, kv.TiFlash) require.Len(t, pushed, len(exprs)) require.Len(t, remained, 0)