From 812789d3d7eca4a07f0f1d83e9b4ed80729b5a13 Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Mon, 1 Jul 2024 15:20:57 +0800 Subject: [PATCH 1/3] This is an automated cherry-pick of #54315 Signed-off-by: ti-chi-bot --- pkg/statistics/BUILD.bazel | 4 + .../handle/history/history_stats.go | 10 +- pkg/statistics/integration_test.go | 125 ++++++++++++++++++ 3 files changed, 133 insertions(+), 6 deletions(-) diff --git a/pkg/statistics/BUILD.bazel b/pkg/statistics/BUILD.bazel index 5c43f8191d883..9f64bc0f86e61 100644 --- a/pkg/statistics/BUILD.bazel +++ b/pkg/statistics/BUILD.bazel @@ -78,7 +78,11 @@ go_test( data = glob(["testdata/**"]), embed = [":statistics"], flaky = True, +<<<<<<< HEAD shard_count = 34, +======= + shard_count = 38, +>>>>>>> 878fa328ea4 (statistics: update stats_history table when it meets duplicate (#54315)) deps = [ "//pkg/config", "//pkg/parser/ast", diff --git a/pkg/statistics/handle/history/history_stats.go b/pkg/statistics/handle/history/history_stats.go index 92f660431448c..ef6d190d24acc 100644 --- a/pkg/statistics/handle/history/history_stats.go +++ b/pkg/statistics/handle/history/history_stats.go @@ -135,10 +135,7 @@ func RecordHistoricalStatsToStorage(sctx sessionctx.Context, physicalID int64, j version = js.Version } else { for _, p := range js.Partitions { - version = p.Version - if version != 0 { - break - } + version = max(version, p.Version) } } blocks, err := storage.JSONTableToBlocks(js, maxColumnSize) @@ -147,9 +144,10 @@ func RecordHistoricalStatsToStorage(sctx sessionctx.Context, physicalID int64, j } ts := time.Now().Format("2006-01-02 15:04:05.999999") - const sql = "INSERT INTO mysql.stats_history(table_id, stats_data, seq_no, version, create_time) VALUES (%?, %?, %?, %?, %?)" + const sql = "INSERT INTO mysql.stats_history(table_id, stats_data, seq_no, version, create_time) VALUES (%?, %?, %?, %?, %?)" + + "ON DUPLICATE KEY UPDATE stats_data=%?, create_time=%?" for i := 0; i < len(blocks); i++ { - if _, err := util.Exec(sctx, sql, physicalID, blocks[i], i, version, ts); err != nil { + if _, err = util.Exec(sctx, sql, physicalID, blocks[i], i, version, ts, blocks[i], ts); err != nil { return 0, errors.Trace(err) } } diff --git a/pkg/statistics/integration_test.go b/pkg/statistics/integration_test.go index 9b51915b174c8..84b199645115e 100644 --- a/pkg/statistics/integration_test.go +++ b/pkg/statistics/integration_test.go @@ -481,3 +481,128 @@ func TestIssue44369(t *testing.T) { tk.MustExec("alter table t rename column b to bb;") tk.MustExec("select * from t where a = 10 and bb > 20;") } +<<<<<<< HEAD +======= + +// Test the case that after ALTER TABLE happens, the pointer to the column info/index info should be refreshed. +func TestColAndIdxExistenceMapChangedAfterAlterTable(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + h := dom.StatsHandle() + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table t(a int, b int, index iab(a,b));") + require.NoError(t, h.HandleDDLEvent(<-h.DDLEventCh())) + tk.MustExec("insert into t value(1,1);") + require.NoError(t, h.DumpStatsDeltaToKV(true)) + tk.MustExec("analyze table t;") + is := dom.InfoSchema() + require.NoError(t, h.Update(is)) + tbl, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tblInfo := tbl.Meta() + statsTbl := h.GetTableStats(tblInfo) + colA := tblInfo.Columns[0] + colInfo := statsTbl.ColAndIdxExistenceMap.GetCol(colA.ID) + require.Equal(t, colA, colInfo) + + tk.MustExec("alter table t modify column a double") + require.NoError(t, h.HandleDDLEvent(<-h.DDLEventCh())) + is = dom.InfoSchema() + require.NoError(t, h.Update(is)) + tbl, err = dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tblInfo = tbl.Meta() + newColA := tblInfo.Columns[0] + require.NotEqual(t, colA.ID, newColA.ID) + statsTbl = h.GetTableStats(tblInfo) + colInfo = statsTbl.ColAndIdxExistenceMap.GetCol(newColA.ID) + require.Equal(t, newColA, colInfo) + tk.MustExec("analyze table t;") + require.NoError(t, h.Update(is)) + statsTbl = h.GetTableStats(tblInfo) + colInfo = statsTbl.ColAndIdxExistenceMap.GetCol(newColA.ID) + require.Equal(t, newColA, colInfo) +} + +func TestTableLastAnalyzeVersion(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + h := dom.StatsHandle() + tk := testkit.NewTestKit(t, store) + + // Only create table should not set the last_analyze_version + tk.MustExec("use test") + tk.MustExec("create table t(a int);") + require.NoError(t, h.HandleDDLEvent(<-h.DDLEventCh())) + is := dom.InfoSchema() + require.NoError(t, h.Update(is)) + tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + statsTbl, found := h.Get(tbl.Meta().ID) + require.True(t, found) + require.Equal(t, uint64(0), statsTbl.LastAnalyzeVersion) + + // Only alter table should not set the last_analyze_version + tk.MustExec("alter table t add column b int default 0") + is = dom.InfoSchema() + tbl, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + require.NoError(t, h.HandleDDLEvent(<-h.DDLEventCh())) + require.NoError(t, h.Update(is)) + statsTbl, found = h.Get(tbl.Meta().ID) + require.True(t, found) + require.Equal(t, uint64(0), statsTbl.LastAnalyzeVersion) + tk.MustExec("alter table t add index idx(a)") + is = dom.InfoSchema() + tbl, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + // We don't handle the ADD INDEX event in the HandleDDLEvent. + require.Equal(t, 0, len(h.DDLEventCh())) + require.NoError(t, err) + require.NoError(t, h.Update(is)) + statsTbl, found = h.Get(tbl.Meta().ID) + require.True(t, found) + require.Equal(t, uint64(0), statsTbl.LastAnalyzeVersion) + + // INSERT and updating the modify_count should not set the last_analyze_version + tk.MustExec("insert into t values(1, 1)") + require.NoError(t, h.DumpStatsDeltaToKV(true)) + require.NoError(t, h.Update(is)) + statsTbl, found = h.Get(tbl.Meta().ID) + require.True(t, found) + require.Equal(t, uint64(0), statsTbl.LastAnalyzeVersion) + + // After analyze, last_analyze_version is set. + tk.MustExec("analyze table t") + require.NoError(t, h.Update(is)) + statsTbl, found = h.Get(tbl.Meta().ID) + require.True(t, found) + require.NotEqual(t, uint64(0), statsTbl.LastAnalyzeVersion) +} + +func TestGlobalIndexWithAnalyzeVersion1AndHistoricalStats(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + + tk.MustExec("set tidb_enable_global_index = true") + tk.MustExec("set tidb_analyze_version = 1") + tk.MustExec("set global tidb_enable_historical_stats = true") + defer tk.MustExec("set global tidb_enable_historical_stats = default") + + tk.MustExec("use test") + tk.MustExec(`CREATE TABLE t ( a int, b int, c int default 0) + PARTITION BY RANGE (a) ( + PARTITION p0 VALUES LESS THAN (10), + PARTITION p1 VALUES LESS THAN (20), + PARTITION p2 VALUES LESS THAN (30), + PARTITION p3 VALUES LESS THAN (40))`) + tk.MustExec("ALTER TABLE t ADD UNIQUE INDEX idx(b)") + tk.MustExec("INSERT INTO t(a, b) values(1, 1), (2, 2), (3, 3), (15, 15), (25, 25), (35, 35)") + + tblID := dom.MustGetTableID(t, "test", "t") + + for i := 0; i < 10; i++ { + tk.MustExec("analyze table t") + } + // Each analyze will only generate one record + tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id=%d", tblID)).Equal(testkit.Rows("10")) +} +>>>>>>> 878fa328ea4 (statistics: update stats_history table when it meets duplicate (#54315)) From f86e5cde2544ad12cfeee0dce22e9ec4f09c96d5 Mon Sep 17 00:00:00 2001 From: Jason Mo Date: Mon, 8 Jul 2024 17:39:41 +0800 Subject: [PATCH 2/3] update --- pkg/statistics/BUILD.bazel | 6 +- pkg/statistics/integration_test.go | 97 ------------------------------ 2 files changed, 1 insertion(+), 102 deletions(-) diff --git a/pkg/statistics/BUILD.bazel b/pkg/statistics/BUILD.bazel index 9f64bc0f86e61..20ee8a66d1270 100644 --- a/pkg/statistics/BUILD.bazel +++ b/pkg/statistics/BUILD.bazel @@ -78,11 +78,7 @@ go_test( data = glob(["testdata/**"]), embed = [":statistics"], flaky = True, -<<<<<<< HEAD - shard_count = 34, -======= - shard_count = 38, ->>>>>>> 878fa328ea4 (statistics: update stats_history table when it meets duplicate (#54315)) + shard_count = 35, deps = [ "//pkg/config", "//pkg/parser/ast", diff --git a/pkg/statistics/integration_test.go b/pkg/statistics/integration_test.go index 84b199645115e..55f06db9943bf 100644 --- a/pkg/statistics/integration_test.go +++ b/pkg/statistics/integration_test.go @@ -481,102 +481,6 @@ func TestIssue44369(t *testing.T) { tk.MustExec("alter table t rename column b to bb;") tk.MustExec("select * from t where a = 10 and bb > 20;") } -<<<<<<< HEAD -======= - -// Test the case that after ALTER TABLE happens, the pointer to the column info/index info should be refreshed. -func TestColAndIdxExistenceMapChangedAfterAlterTable(t *testing.T) { - store, dom := testkit.CreateMockStoreAndDomain(t) - h := dom.StatsHandle() - tk := testkit.NewTestKit(t, store) - tk.MustExec("use test") - tk.MustExec("create table t(a int, b int, index iab(a,b));") - require.NoError(t, h.HandleDDLEvent(<-h.DDLEventCh())) - tk.MustExec("insert into t value(1,1);") - require.NoError(t, h.DumpStatsDeltaToKV(true)) - tk.MustExec("analyze table t;") - is := dom.InfoSchema() - require.NoError(t, h.Update(is)) - tbl, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) - require.NoError(t, err) - tblInfo := tbl.Meta() - statsTbl := h.GetTableStats(tblInfo) - colA := tblInfo.Columns[0] - colInfo := statsTbl.ColAndIdxExistenceMap.GetCol(colA.ID) - require.Equal(t, colA, colInfo) - - tk.MustExec("alter table t modify column a double") - require.NoError(t, h.HandleDDLEvent(<-h.DDLEventCh())) - is = dom.InfoSchema() - require.NoError(t, h.Update(is)) - tbl, err = dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) - require.NoError(t, err) - tblInfo = tbl.Meta() - newColA := tblInfo.Columns[0] - require.NotEqual(t, colA.ID, newColA.ID) - statsTbl = h.GetTableStats(tblInfo) - colInfo = statsTbl.ColAndIdxExistenceMap.GetCol(newColA.ID) - require.Equal(t, newColA, colInfo) - tk.MustExec("analyze table t;") - require.NoError(t, h.Update(is)) - statsTbl = h.GetTableStats(tblInfo) - colInfo = statsTbl.ColAndIdxExistenceMap.GetCol(newColA.ID) - require.Equal(t, newColA, colInfo) -} - -func TestTableLastAnalyzeVersion(t *testing.T) { - store, dom := testkit.CreateMockStoreAndDomain(t) - h := dom.StatsHandle() - tk := testkit.NewTestKit(t, store) - - // Only create table should not set the last_analyze_version - tk.MustExec("use test") - tk.MustExec("create table t(a int);") - require.NoError(t, h.HandleDDLEvent(<-h.DDLEventCh())) - is := dom.InfoSchema() - require.NoError(t, h.Update(is)) - tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) - require.NoError(t, err) - statsTbl, found := h.Get(tbl.Meta().ID) - require.True(t, found) - require.Equal(t, uint64(0), statsTbl.LastAnalyzeVersion) - - // Only alter table should not set the last_analyze_version - tk.MustExec("alter table t add column b int default 0") - is = dom.InfoSchema() - tbl, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) - require.NoError(t, err) - require.NoError(t, h.HandleDDLEvent(<-h.DDLEventCh())) - require.NoError(t, h.Update(is)) - statsTbl, found = h.Get(tbl.Meta().ID) - require.True(t, found) - require.Equal(t, uint64(0), statsTbl.LastAnalyzeVersion) - tk.MustExec("alter table t add index idx(a)") - is = dom.InfoSchema() - tbl, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) - // We don't handle the ADD INDEX event in the HandleDDLEvent. - require.Equal(t, 0, len(h.DDLEventCh())) - require.NoError(t, err) - require.NoError(t, h.Update(is)) - statsTbl, found = h.Get(tbl.Meta().ID) - require.True(t, found) - require.Equal(t, uint64(0), statsTbl.LastAnalyzeVersion) - - // INSERT and updating the modify_count should not set the last_analyze_version - tk.MustExec("insert into t values(1, 1)") - require.NoError(t, h.DumpStatsDeltaToKV(true)) - require.NoError(t, h.Update(is)) - statsTbl, found = h.Get(tbl.Meta().ID) - require.True(t, found) - require.Equal(t, uint64(0), statsTbl.LastAnalyzeVersion) - - // After analyze, last_analyze_version is set. - tk.MustExec("analyze table t") - require.NoError(t, h.Update(is)) - statsTbl, found = h.Get(tbl.Meta().ID) - require.True(t, found) - require.NotEqual(t, uint64(0), statsTbl.LastAnalyzeVersion) -} func TestGlobalIndexWithAnalyzeVersion1AndHistoricalStats(t *testing.T) { store, dom := testkit.CreateMockStoreAndDomain(t) @@ -605,4 +509,3 @@ func TestGlobalIndexWithAnalyzeVersion1AndHistoricalStats(t *testing.T) { // Each analyze will only generate one record tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id=%d", tblID)).Equal(testkit.Rows("10")) } ->>>>>>> 878fa328ea4 (statistics: update stats_history table when it meets duplicate (#54315)) From d21b5976148bfb67849784c4e9d17e396c3934fa Mon Sep 17 00:00:00 2001 From: Jason Mo Date: Mon, 8 Jul 2024 17:53:15 +0800 Subject: [PATCH 3/3] update --- pkg/statistics/integration_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/statistics/integration_test.go b/pkg/statistics/integration_test.go index 55f06db9943bf..c675cd47f857c 100644 --- a/pkg/statistics/integration_test.go +++ b/pkg/statistics/integration_test.go @@ -23,6 +23,7 @@ import ( "time" "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze" @@ -483,10 +484,13 @@ func TestIssue44369(t *testing.T) { } func TestGlobalIndexWithAnalyzeVersion1AndHistoricalStats(t *testing.T) { + defer config.RestoreFunc()() + config.UpdateGlobal(func(conf *config.Config) { + conf.EnableGlobalIndex = true + }) store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) - tk.MustExec("set tidb_enable_global_index = true") tk.MustExec("set tidb_analyze_version = 1") tk.MustExec("set global tidb_enable_historical_stats = true") defer tk.MustExec("set global tidb_enable_historical_stats = default")