From 1577ca2613b5e63fb0fb3f936019d88ecf2aa23b Mon Sep 17 00:00:00 2001 From: AntiTopQuark Date: Tue, 9 Aug 2022 08:45:55 +0800 Subject: [PATCH 1/5] support castTimeAsDuration in tidb --- expression/expr_to_pb_test.go | 5 +++++ expression/expression.go | 2 ++ planner/core/integration_test.go | 36 ++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/expression/expr_to_pb_test.go b/expression/expr_to_pb_test.go index 7b49a4807b71e..b817da970ceb9 100644 --- a/expression/expr_to_pb_test.go +++ b/expression/expr_to_pb_test.go @@ -656,6 +656,11 @@ func TestExprPushDownToFlash(t *testing.T) { require.NoError(t, err) exprs = append(exprs, function) + // CastTimeAsDuration + function, err = NewFunction(mock.NewContext(), ast.Cast, types.NewFieldType(mysql.TypeDatetime), durationColumn) + require.NoError(t, err) + exprs = append(exprs, function) + // Substring2ArgsUTF8 function, err = NewFunction(mock.NewContext(), ast.Substr, types.NewFieldType(mysql.TypeString), stringColumn, intColumn) require.NoError(t, err) diff --git a/expression/expression.go b/expression/expression.go index fb4b870e8263a..0f7df4c38e565 100644 --- a/expression/expression.go +++ b/expression/expression.go @@ -1105,6 +1105,8 @@ func scalarExprSupportedByFlash(function *ScalarFunction) bool { tipb.ScalarFuncSig_CastStringAsTime /*, tipb.ScalarFuncSig_CastDurationAsTime, tipb.ScalarFuncSig_CastJsonAsTime*/ : // ban the function of casting year type as time type pushing down to tiflash because of https://github.com/pingcap/tidb/issues/26215 return function.GetArgs()[0].GetType().GetType() != mysql.TypeYear + case tipb.ScalarFuncSig_CastTimeAsDuration: + return sourceType.GetType() == retType.GetType() || retType.GetType() == mysql.TypeDuration } case ast.DateAdd, ast.AddDate: switch function.Function.PbCode() { diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 73d6292cb925f..9b4da47ee6f05 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -7024,3 +7024,39 @@ func TestIssue36609(t *testing.T) { tk.MustQuery("select * from t3 straight_join t4 on t3.a = t4.b straight_join t2 on t3.d = t2.c straight_join t1 on t1.a = t2.b straight_join t5 on t4.c = t5.d where t2.b < 100 and t4.a = 10;") tk.MustQuery("select * from information_schema.statements_summary;") } + +func TestLeftShiftPushDownToTiFlash(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table test.t(c1 date, c2 datetime(4))") + tk.MustExec("insert into test.t values('2022-01-01','2022-09-20 11:11:11.0000')") + tk.MustExec("insert into test.t values('2022-01-01','2022-09-20 11:11:11.1111')") + tk.MustExec("insert into test.t values('2022-01-01','2022-09-20 11:11:11.1234')") + tk.MustExec("insert into test.t values('2022-01-01','2022-09-20 11:11:11.1255')") + tk.MustExec("insert into test.t values('2022-01-01','2022-09-20 11:11:11.9999')") + tk.MustExec("set @@tidb_allow_mpp=1; set @@tidb_enforce_mpp=1") + tk.MustExec("set @@tidb_isolation_read_engines = 'tiflash'") + + // Create virtual tiflash replica info. + is := dom.InfoSchema() + db, exists := is.SchemaByName(model.NewCIStr("test")) + require.True(t, exists) + for _, tblInfo := range db.Tables { + if tblInfo.Name.L == "t" { + tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{ + Count: 1, + Available: true, + } + } + } + + rows := [][]interface{}{ + {"TableReader_9", "root", "data:ExchangeSender_8"}, + {"└─ExchangeSender_8", "mpp[tiflash]", "ExchangeType: PassThrough"}, + {" └─Projection_4", "mpp[tiflash]", "cast(test.t.c2, time(2))->Column#4"}, + {" └─TableFullScan_7", "mpp[tiflash]", "keep order:false, stats:pseudo"}, + } + tk.MustQuery("explain select cast(test.t.c2 as time(2)) from test.t;").CheckAt([]int{0, 2, 4}, rows) +} \ No newline at end of file From 49f4afd0828a130052f391d9cba36261220cd802 Mon Sep 17 00:00:00 2001 From: AntiTopQuark Date: Wed, 10 Aug 2022 08:51:49 +0800 Subject: [PATCH 2/5] update integration_test --- planner/core/integration_test.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 9b4da47ee6f05..43daf63b4e5bc 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -7025,17 +7025,18 @@ func TestIssue36609(t *testing.T) { tk.MustQuery("select * from information_schema.statements_summary;") } -func TestLeftShiftPushDownToTiFlash(t *testing.T) { +func TestCastTimeAsDurationToTiFlash(t *testing.T) { store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") tk.MustExec("drop table if exists t") - tk.MustExec("create table test.t(c1 date, c2 datetime(4))") - tk.MustExec("insert into test.t values('2022-01-01','2022-09-20 11:11:11.0000')") - tk.MustExec("insert into test.t values('2022-01-01','2022-09-20 11:11:11.1111')") - tk.MustExec("insert into test.t values('2022-01-01','2022-09-20 11:11:11.1234')") - tk.MustExec("insert into test.t values('2022-01-01','2022-09-20 11:11:11.1255')") - tk.MustExec("insert into test.t values('2022-01-01','2022-09-20 11:11:11.9999')") + tk.MustExec("create table t(a date, b datetime(4))") + tk.MustExec("insert into t values('2021-10-26', '2021-10-26')") + tk.MustExec("insert into t values('2021-10-26', '2021-10-26 11:11:11')") + tk.MustExec("insert into t values('2021-10-26', '2021-10-26 11:11:11.111111')") + tk.MustExec("insert into t values('2021-10-26', '2021-10-26 11:11:11.123456')") + tk.MustExec("insert into t values('2021-10-26', '2021-10-26 11:11:11.999999')") + tk.MustExec("set @@tidb_allow_mpp=1; set @@tidb_enforce_mpp=1") tk.MustExec("set @@tidb_isolation_read_engines = 'tiflash'") @@ -7055,8 +7056,8 @@ func TestLeftShiftPushDownToTiFlash(t *testing.T) { rows := [][]interface{}{ {"TableReader_9", "root", "data:ExchangeSender_8"}, {"└─ExchangeSender_8", "mpp[tiflash]", "ExchangeType: PassThrough"}, - {" └─Projection_4", "mpp[tiflash]", "cast(test.t.c2, time(2))->Column#4"}, + {" └─Projection_4", "mpp[tiflash]", "cast(test.t.b, time BINARY)->Column#4"}, {" └─TableFullScan_7", "mpp[tiflash]", "keep order:false, stats:pseudo"}, } - tk.MustQuery("explain select cast(test.t.c2 as time(2)) from test.t;").CheckAt([]int{0, 2, 4}, rows) + tk.MustQuery("explain select cast(b as time) from t;").CheckAt([]int{0, 2, 4}, rows) } \ No newline at end of file From e5b2c4223506bd385d178448c1b7b5c788b284fb Mon Sep 17 00:00:00 2001 From: AntiTopQuark Date: Thu, 11 Aug 2022 15:24:52 +0800 Subject: [PATCH 3/5] expression: support to pushdown castTimeAsDuration to TiFlash 1. update the conditions for castTimeAsDuration 2. update integration test --- expression/expression.go | 2 +- planner/core/integration_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/expression/expression.go b/expression/expression.go index 2051256365cd3..30bff1815ff30 100644 --- a/expression/expression.go +++ b/expression/expression.go @@ -1106,7 +1106,7 @@ func scalarExprSupportedByFlash(function *ScalarFunction) bool { // ban the function of casting year type as time type pushing down to tiflash because of https://github.com/pingcap/tidb/issues/26215 return function.GetArgs()[0].GetType().GetType() != mysql.TypeYear case tipb.ScalarFuncSig_CastTimeAsDuration: - return sourceType.GetType() == retType.GetType() || retType.GetType() == mysql.TypeDuration + return retType.GetType() == mysql.TypeDuration } case ast.DateAdd, ast.AddDate: switch function.Function.PbCode() { diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 0ca51106b00c0..27d41d130a245 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -7123,8 +7123,8 @@ func TestCastTimeAsDurationToTiFlash(t *testing.T) { rows := [][]interface{}{ {"TableReader_9", "root", "data:ExchangeSender_8"}, {"└─ExchangeSender_8", "mpp[tiflash]", "ExchangeType: PassThrough"}, - {" └─Projection_4", "mpp[tiflash]", "cast(test.t.b, time BINARY)->Column#4"}, + {" └─Projection_4", "mpp[tiflash]", " cast(test.t.a, time BINARY)->Column#4, cast(test.t.b, time BINARY)->Column#5"}, {" └─TableFullScan_7", "mpp[tiflash]", "keep order:false, stats:pseudo"}, } - tk.MustQuery("explain select cast(b as time) from t;").CheckAt([]int{0, 2, 4}, rows) + tk.MustQuery("explain select cast(a as time), cast(b as time) from t;").CheckAt([]int{0, 2, 4}, rows) } From 5d67bd0d4e634502dc6d3cfa470e981d8223d4f4 Mon Sep 17 00:00:00 2001 From: AntiTopQuark Date: Fri, 12 Aug 2022 08:42:14 +0800 Subject: [PATCH 4/5] update integration test --- planner/core/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 9834057ffa469..a91bc16fe38e2 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -7124,7 +7124,7 @@ func TestCastTimeAsDurationToTiFlash(t *testing.T) { rows := [][]interface{}{ {"TableReader_9", "root", "data:ExchangeSender_8"}, {"└─ExchangeSender_8", "mpp[tiflash]", "ExchangeType: PassThrough"}, - {" └─Projection_4", "mpp[tiflash]", " cast(test.t.a, time BINARY)->Column#4, cast(test.t.b, time BINARY)->Column#5"}, + {" └─Projection_4", "mpp[tiflash]", "cast(test.t.a, time BINARY)->Column#4, cast(test.t.b, time BINARY)->Column#5"}, {" └─TableFullScan_7", "mpp[tiflash]", "keep order:false, stats:pseudo"}, } tk.MustQuery("explain select cast(a as time), cast(b as time) from t;").CheckAt([]int{0, 2, 4}, rows) From 4a69acf49016c981463491e5610cf6c590c55e0a Mon Sep 17 00:00:00 2001 From: AntiTopQuark Date: Fri, 12 Aug 2022 09:17:06 +0800 Subject: [PATCH 5/5] update expr to pb test --- expression/expr_to_pb_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/expression/expr_to_pb_test.go b/expression/expr_to_pb_test.go index 519673acb520b..af1e0471260d1 100644 --- a/expression/expr_to_pb_test.go +++ b/expression/expr_to_pb_test.go @@ -656,11 +656,6 @@ func TestExprPushDownToFlash(t *testing.T) { require.NoError(t, err) exprs = append(exprs, function) - // CastTimeAsDuration - function, err = NewFunction(mock.NewContext(), ast.Cast, types.NewFieldType(mysql.TypeDatetime), durationColumn) - require.NoError(t, err) - exprs = append(exprs, function) - // Substring2ArgsUTF8 function, err = NewFunction(mock.NewContext(), ast.Substr, types.NewFieldType(mysql.TypeString), stringColumn, intColumn) require.NoError(t, err) @@ -1190,6 +1185,11 @@ func TestExprPushDownToFlash(t *testing.T) { pushed, remained = PushDownExprsWithExtraInfo(sc, exprs, client, kv.TiFlash, true) require.Len(t, pushed, len(exprs)) require.Len(t, remained, 0) + + // CastTimeAsDuration + function, err = NewFunction(mock.NewContext(), ast.Cast, types.NewFieldType(mysql.TypeDatetime), durationColumn) + require.NoError(t, err) + exprs = append(exprs, function) } func TestExprOnlyPushDownToFlash(t *testing.T) {