From 97bbab03f65175ef013a255301b4008aef738308 Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Tue, 8 Mar 2022 13:23:48 +0100 Subject: [PATCH] cherry pick #32721 to release-5.1 Signed-off-by: ti-srebot --- planner/core/partition_pruner_test.go | 128 +++++++++++++++++++++-- planner/core/rule_partition_processor.go | 2 +- 2 files changed, 118 insertions(+), 12 deletions(-) diff --git a/planner/core/partition_pruner_test.go b/planner/core/partition_pruner_test.go index 07dee6f73741e..76886387f70a1 100644 --- a/planner/core/partition_pruner_test.go +++ b/planner/core/partition_pruner_test.go @@ -145,21 +145,127 @@ func (s *testPartitionPruneSuit) TestRangeColumnPartitionPruningForIn(c *C) { " └─Selection 20.00 cop[tikv] in(test_range_col_in.t2.a, -1, 1)", " └─TableFullScan 10000.00 cop[tikv] table:t2, partition:p1 keep order:false, stats:pseudo")) - // for other types, the in-pruning shouldn't be working for safety tk.MustExec(`create table t3 (a varchar(10)) partition by range columns(a) ( partition p0 values less than ("aaa"), partition p1 values less than ("bbb"), partition p2 values less than ("ccc"))`) - tk.MustQuery(`explain format='brief' select a from t3 where a in ('aaa', 'aab')`).Check(testkit.Rows("PartitionUnion 60.00 root ", - "├─TableReader 20.00 root data:Selection", - "│ └─Selection 20.00 cop[tikv] in(test_range_col_in.t3.a, \"aaa\", \"aab\")", - "│ └─TableFullScan 10000.00 cop[tikv] table:t3, partition:p0 keep order:false, stats:pseudo", - "├─TableReader 20.00 root data:Selection", - "│ └─Selection 20.00 cop[tikv] in(test_range_col_in.t3.a, \"aaa\", \"aab\")", - "│ └─TableFullScan 10000.00 cop[tikv] table:t3, partition:p1 keep order:false, stats:pseudo", - "└─TableReader 20.00 root data:Selection", - " └─Selection 20.00 cop[tikv] in(test_range_col_in.t3.a, \"aaa\", \"aab\")", - " └─TableFullScan 10000.00 cop[tikv] table:t3, partition:p2 keep order:false, stats:pseudo")) + tk.MustQuery(`explain format='brief' select a from t3 where a in ('aaa', 'aab')`).Check(testkit.Rows( + `TableReader 20.00 root data:Selection`, + `└─Selection 20.00 cop[tikv] in(test_range_col_in.t3.a, "aaa", "aab")`, + ` └─TableFullScan 10000.00 cop[tikv] table:t3, partition:p1 keep order:false, stats:pseudo`)) + tk.MustQuery(`explain format='brief' select a from t3 where a in ('aaa', 'bu')`).Check(testkit.Rows( + `PartitionUnion 40.00 root `, + `├─TableReader 20.00 root data:Selection`, + `│ └─Selection 20.00 cop[tikv] in(test_range_col_in.t3.a, "aaa", "bu")`, + `│ └─TableFullScan 10000.00 cop[tikv] table:t3, partition:p1 keep order:false, stats:pseudo`, + `└─TableReader 20.00 root data:Selection`, + ` └─Selection 20.00 cop[tikv] in(test_range_col_in.t3.a, "aaa", "bu")`, + ` └─TableFullScan 10000.00 cop[tikv] table:t3, partition:p2 keep order:false, stats:pseudo`)) +} + +func TestRangeColumnPartitionPruningForInString(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + tk := testkit.NewTestKit(t, store) + tk.MustExec("drop database if exists test_range_col_in_string") + tk.MustExec("create database test_range_col_in_string") + + tk.MustExec("use test_range_col_in_string") + tk.MustExec("set names utf8mb4 collate utf8mb4_bin") + tk.MustExec("set @@session.tidb_partition_prune_mode='static'") + + type testStruct struct { + sql string + partitions string + rows []string + } + + extractPartitions := func(res *testkit.Result) string { + planStrings := testdata.ConvertRowsToStrings(res.Rows()) + partitions := []string{} + for _, s := range planStrings { + parts := getFieldValue("partition:", s) + if parts != "" { + partitions = append(partitions, strings.Split(parts, ",")...) + } + } + return strings.Join(partitions, ",") + } + checkColumnStringPruningTests := func(tests []testStruct) { + modes := []string{"dynamic", "static"} + for _, mode := range modes { + tk.MustExec(`set @@tidb_partition_prune_mode = '` + mode + `'`) + for _, test := range tests { + explainResult := tk.MustQuery("explain format = 'brief' " + test.sql) + partitions := strings.ToLower(extractPartitions(explainResult)) + require.Equal(t, test.partitions, partitions, "Mode: %s sql: %s", mode, test.sql) + tk.MustQuery(test.sql).Sort().Check(testkit.Rows(test.rows...)) + } + } + } + tk.MustExec("create table t (a varchar(255) charset utf8mb4 collate utf8mb4_bin) partition by range columns(a)" + + `( partition pNull values less than (""),` + + `partition pAAAA values less than ("AAAA"),` + + `partition pCCC values less than ("CCC"),` + + `partition pShrimpsandwich values less than ("Räksmörgås"),` + + `partition paaa values less than ("aaa"),` + + `partition pSushi values less than ("🍣🍣🍣"),` + + `partition pMax values less than (MAXVALUE))`) + tk.MustExec(`insert into t values (NULL), ("a"), ("Räkmacka"), ("🍣 is life"), ("🍺 after work?"), ("🍺🍺🍺🍺🍺 for oktoberfest"),("AA"),("aa"),("AAA"),("aaa")`) + tests := []testStruct{ + // Lower case partition names due to issue#32719 + {sql: `select * from t where a IS NULL`, partitions: "pnull", rows: []string{""}}, + {sql: `select * from t where a = 'AA'`, partitions: "paaaa", rows: []string{"AA"}}, + {sql: `select * from t where a = 'AA' collate utf8mb4_general_ci`, partitions: "paaaa", rows: []string{"AA"}}, // Notice that the it not uses _bin collation for partition => 'aa' not found! #32749 + {sql: `select * from t where a = 'aa'`, partitions: "paaa", rows: []string{"aa"}}, + {sql: `select * from t where a = 'aa' collate utf8mb4_general_ci`, partitions: "paaaa", rows: []string{"AA"}}, // Notice that the it not uses _bin collation for partition => 'aa' not found! #32749 + {sql: `select * from t where a = 'AAA'`, partitions: "paaaa", rows: []string{"AAA"}}, + {sql: `select * from t where a = 'AB'`, partitions: "pccc", rows: []string{}}, + {sql: `select * from t where a = 'aB'`, partitions: "paaa", rows: []string{}}, + {sql: `select * from t where a = '🍣'`, partitions: "psushi", rows: []string{}}, + {sql: `select * from t where a in ('🍣 is life', "Räkmacka", "🍺🍺🍺🍺 after work?")`, partitions: "pshrimpsandwich,psushi,pmax", rows: []string{"Räkmacka", "🍣 is life"}}, + {sql: `select * from t where a in ('AAA', 'aa')`, partitions: "paaaa,paaa", rows: []string{"AAA", "aa"}}, + {sql: `select * from t where a in ('AAA' collate utf8mb4_general_ci, 'aa')`, partitions: "paaaa,paaa", rows: []string{"AA", "AAA", "aa"}}, // aaa missing due to #32749 + {sql: `select * from t where a in ('AAA', 'aa' collate utf8mb4_general_ci)`, partitions: "paaaa", rows: []string{"AA", "AAA"}}, // aa, aaa missing due to #32749 + } + checkColumnStringPruningTests(tests) + tk.MustExec(`set names utf8mb4 collate utf8mb4_general_ci`) + checkColumnStringPruningTests(tests) + tk.MustExec(`set names utf8mb4 collate utf8mb4_unicode_ci`) + checkColumnStringPruningTests(tests) + tk.MustExec("drop table t") + tk.MustExec("create table t (a varchar(255) charset utf8mb4 collate utf8mb4_general_ci) partition by range columns(a)" + + `( partition pNull values less than (""),` + + `partition paaa values less than ("aaa"),` + + `partition pAAAA values less than ("AAAA"),` + + `partition pCCC values less than ("CCC"),` + + `partition pShrimpsandwich values less than ("Räksmörgås"),` + + `partition pSushi values less than ("🍣🍣🍣"),` + + `partition pMax values less than (MAXVALUE))`) + tk.MustExec(`insert into t values (NULL), ("a"), ("Räkmacka"), ("🍣 is life"), ("🍺 after work?"), ("🍺🍺🍺🍺🍺 for oktoberfest"),("AA"),("aa"),("AAA"),("aaa")`) + + tests = []testStruct{ + // Lower case partition names due to issue#32719 + {sql: `select * from t where a IS NULL`, partitions: "pnull", rows: []string{""}}, + {sql: `select * from t where a = 'AA'`, partitions: "paaa", rows: []string{"AA", "aa"}}, + {sql: `select * from t where a = 'AA' collate utf8mb4_bin`, partitions: "paaa", rows: []string{"AA"}}, + {sql: `select * from t where a = 'AAA'`, partitions: "paaaa", rows: []string{"AAA", "aaa"}}, + {sql: `select * from t where a = 'AAA' collate utf8mb4_bin`, partitions: "paaa", rows: []string{}}, // Notice that the it uses _bin collation for partition => not found! #32749 + {sql: `select * from t where a = 'AB'`, partitions: "pccc", rows: []string{}}, + {sql: `select * from t where a = 'aB'`, partitions: "pccc", rows: []string{}}, + {sql: `select * from t where a = '🍣'`, partitions: "psushi", rows: []string{}}, + {sql: `select * from t where a in ('🍣 is life', "Räkmacka", "🍺🍺🍺🍺 after work?")`, partitions: "pshrimpsandwich,psushi,pmax", rows: []string{"Räkmacka", "🍣 is life"}}, + {sql: `select * from t where a in ('AA', 'aaa')`, partitions: "paaa,paaaa", rows: []string{"AA", "AAA", "aa", "aaa"}}, + {sql: `select * from t where a in ('AAA' collate utf8mb4_bin, 'aa')`, partitions: "paaa", rows: []string{"aa"}}, // AAA missing due to #32749, why is AA missing? + {sql: `select * from t where a in ('AAA', 'aa' collate utf8mb4_bin)`, partitions: "paaaa,psushi", rows: []string{"AAA"}}, // aa, aaa missing due to #32749 also all missing paaa + } + + tk.MustExec(`set names utf8mb4 collate utf8mb4_bin`) + checkColumnStringPruningTests(tests) + tk.MustExec(`set names utf8mb4 collate utf8mb4_general_ci`) + checkColumnStringPruningTests(tests) + tk.MustExec(`set names utf8mb4 collate utf8mb4_unicode_ci`) + checkColumnStringPruningTests(tests) } func (s *testPartitionPruneSuit) TestListPartitionPruner(c *C) { diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index 8e42e4cadf916..1fc2376d97f3f 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -1031,7 +1031,7 @@ func partitionRangeColumnForInExpr(sctx sessionctx.Context, args []expression.Ex return pruner.fullRange() } switch constExpr.Value.Kind() { - case types.KindInt64, types.KindUint64, types.KindMysqlTime: // for safety, only support int and datetime now + case types.KindInt64, types.KindUint64, types.KindMysqlTime, types.KindString: // for safety, only support string,int and datetime now case types.KindNull: result = append(result, partitionRange{0, 1}) continue