From cff890a3d90f3b0b1bcebfd69b58ae5ebf1ee1c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E8=B6=85?= Date: Mon, 31 Oct 2022 14:17:59 +0800 Subject: [PATCH 01/13] *: add server ip in connection info (#38723) close pingcap/tidb#38493 --- server/conn.go | 11 +++++++++++ server/server.go | 1 + server/tidb_test.go | 4 ++++ sessionctx/variable/session.go | 1 + 4 files changed, 17 insertions(+) diff --git a/server/conn.go b/server/conn.go index 8065c451e73e5..c2e3f1db9b299 100644 --- a/server/conn.go +++ b/server/conn.go @@ -197,6 +197,7 @@ type clientConn struct { *TiDBContext // an interface to execute sql statements. } attrs map[string]string // attributes parsed from client handshake response, not used for now. + serverHost string // server host peerHost string // peer host peerPort string // peer port status int32 // dispatching/reading/shutdown/waitshutdown @@ -949,6 +950,7 @@ func (cc *clientConn) PeerHost(hasPassword string) (host, port string, err error host = variable.DefHostname if cc.isUnixSocket { cc.peerHost = host + cc.serverHost = host return } addr := cc.bufReadConn.RemoteAddr().String() @@ -959,6 +961,15 @@ func (cc *clientConn) PeerHost(hasPassword string) (host, port string, err error } cc.peerHost = host cc.peerPort = port + + serverAddr := cc.bufReadConn.LocalAddr().String() + serverHost, _, err := net.SplitHostPort(serverAddr) + if err != nil { + err = errAccessDenied.GenWithStackByArgs(cc.user, addr, hasPassword) + return + } + cc.serverHost = serverHost + return } diff --git a/server/server.go b/server/server.go index da79b49e114a2..9ed4b963913ec 100644 --- a/server/server.go +++ b/server/server.go @@ -625,6 +625,7 @@ func (cc *clientConn) connectInfo() *variable.ConnectionInfo { ClientIP: cc.peerHost, ClientPort: cc.peerPort, ServerID: 1, + ServerIP: cc.serverHost, ServerPort: int(cc.server.cfg.Port), User: cc.user, ServerOSLoginUser: osUser, diff --git a/server/tidb_test.go b/server/tidb_test.go index 6e2f3d5ef4a7a..293406fe87538 100644 --- a/server/tidb_test.go +++ b/server/tidb_test.go @@ -2836,6 +2836,8 @@ func TestExtensionConnEvent(t *testing.T) { conn2.User = "root" conn2.DB = "test" + require.Equal(t, "127.0.0.1", conn1.ClientIP) + require.Equal(t, "127.0.0.1", conn1.ServerIP) require.Empty(t, conn1.User) require.Empty(t, conn1.DB) require.Equal(t, conn2, logs.infos[1]) @@ -2874,6 +2876,8 @@ func TestExtensionConnEvent(t *testing.T) { conn2.User = "noexist" conn2.DB = "test" + require.Equal(t, "127.0.0.1", conn1.ClientIP) + require.Equal(t, "127.0.0.1", conn1.ServerIP) require.Empty(t, conn1.User) require.Empty(t, conn1.DB) require.Equal(t, conn2, logs.infos[1]) diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index f7f875e50fb93..94e5fcddc5968 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1440,6 +1440,7 @@ type ConnectionInfo struct { ClientIP string ClientPort string ServerID int + ServerIP string ServerPort int Duration float64 User string From 65a318fcb5ea6f7790fc65439faf182d2e579794 Mon Sep 17 00:00:00 2001 From: crazycs Date: Mon, 31 Oct 2022 14:35:59 +0800 Subject: [PATCH 02/13] executor: add memory track in handle cop request (#38755) close pingcap/tidb#33893 --- executor/coprocessor.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/executor/coprocessor.go b/executor/coprocessor.go index 93a23dd6829ca..184aa0d9945f1 100644 --- a/executor/coprocessor.go +++ b/executor/coprocessor.go @@ -62,6 +62,7 @@ func (h *CoprocessorDAGHandler) HandleRequest(ctx context.Context, req *coproces chk := newFirstChunk(e) tps := e.base().retFieldTypes var totalChunks, partChunks []tipb.Chunk + memTracker := h.sctx.GetSessionVars().StmtCtx.MemTracker for { chk.Reset() err = Next(ctx, e, chk) @@ -75,6 +76,9 @@ func (h *CoprocessorDAGHandler) HandleRequest(ctx context.Context, req *coproces if err != nil { return h.buildErrorResponse(err) } + for _, ch := range partChunks { + memTracker.Consume(int64(ch.Size())) + } totalChunks = append(totalChunks, partChunks...) } if err := e.Close(); err != nil { From a151383acf66cb9eff1d7474140bac9f420d4ced Mon Sep 17 00:00:00 2001 From: Meng Xin Date: Mon, 31 Oct 2022 17:57:59 +0800 Subject: [PATCH 03/13] util: fix panic cause by nil plan in memeory usage alarm (#38727) --- session/session.go | 6 ++++++ util/memoryusagealarm/memoryusagealarm.go | 2 +- util/processinfo.go | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/session/session.go b/session/session.go index f3bd34f12ea47..f43e265462c11 100644 --- a/session/session.go +++ b/session/session.go @@ -1563,6 +1563,10 @@ func (s *session) SetProcessInfo(sql string, t time.Time, command byte, maxExecu if explain, ok := p.(*plannercore.Explain); ok && explain.Analyze && explain.TargetPlan != nil { p = explain.TargetPlan } + canExplainAnalyze := false + if _, ok := p.(plannercore.PhysicalPlan); ok { + canExplainAnalyze = true + } pi := util.ProcessInfo{ ID: s.sessionVars.ConnectionID, Port: s.sessionVars.Port, @@ -1581,6 +1585,7 @@ func (s *session) SetProcessInfo(sql string, t time.Time, command byte, maxExecu StatsInfo: plannercore.GetStatsInfo, MaxExecutionTime: maxExecutionTime, RedactSQL: s.sessionVars.EnableRedactLog, + CanExplainAnalyze: canExplainAnalyze, } oldPi := s.ShowProcess() if p == nil { @@ -1590,6 +1595,7 @@ func (s *session) SetProcessInfo(sql string, t time.Time, command byte, maxExecu pi.Plan = oldPi.Plan pi.PlanExplainRows = oldPi.PlanExplainRows pi.RuntimeStatsColl = oldPi.RuntimeStatsColl + _, pi.CanExplainAnalyze = pi.Plan.(plannercore.PhysicalPlan) } } // We set process info before building plan, so we extended execution time. diff --git a/util/memoryusagealarm/memoryusagealarm.go b/util/memoryusagealarm/memoryusagealarm.go index 0d68e16810ff1..afa4a75a19260 100644 --- a/util/memoryusagealarm/memoryusagealarm.go +++ b/util/memoryusagealarm/memoryusagealarm.go @@ -315,7 +315,7 @@ func (record *memoryUsageAlarm) recordSQL(sm util.SessionManager, recordDir stri processInfo := sm.ShowProcessList() pinfo := make([]*util.ProcessInfo, 0, len(processInfo)) for _, info := range processInfo { - if len(info.Info) != 0 { + if len(info.Info) != 0 && info.CanExplainAnalyze { pinfo = append(pinfo, info) } } diff --git a/util/processinfo.go b/util/processinfo.go index 9b0066d868c79..f70c324178e4a 100644 --- a/util/processinfo.go +++ b/util/processinfo.go @@ -61,6 +61,7 @@ type ProcessInfo struct { Command byte ExceedExpensiveTimeThresh bool RedactSQL bool + CanExplainAnalyze bool } // ToRowForShow returns []interface{} for the row data of "SHOW [FULL] PROCESSLIST". From b7aae15952d2998ae29801d3b0652a6099af3f9e Mon Sep 17 00:00:00 2001 From: ekexium Date: Mon, 31 Oct 2022 18:25:59 +0800 Subject: [PATCH 04/13] txn: fix false assertion error caused by loss of pessimistic locks (#38392) close pingcap/tidb#38365 --- DEPS.bzl | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- tests/realtikvtest/txntest/txn_test.go | 23 +++++++++++++++++++++++ 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/DEPS.bzl b/DEPS.bzl index a65cf59a148bf..e9d0d04876024 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -3429,8 +3429,8 @@ def go_deps(): name = "com_github_tikv_client_go_v2", build_file_proto_mode = "disable_global", importpath = "github.com/tikv/client-go/v2", - sum = "h1:5KLqhDGLc/mtemdS/odfOP717rn8ttsTj3jzZ8TZn9A=", - version = "v2.0.1-0.20221017092635-91be9c6ce6c0", + sum = "h1:s8eJEGI4p/fxFwMBkoJ+4FAEQNQhHR47TZmVW+EEtOE=", + version = "v2.0.1-0.20221026083454-6c9c7c7c5815", ) go_repository( name = "com_github_tikv_pd_client", diff --git a/go.mod b/go.mod index 458644b15790c..0b7ff4a451cdc 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,7 @@ require ( github.com/stretchr/testify v1.8.0 github.com/tdakkota/asciicheck v0.1.1 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 - github.com/tikv/client-go/v2 v2.0.1-0.20221017092635-91be9c6ce6c0 + github.com/tikv/client-go/v2 v2.0.1-0.20221026083454-6c9c7c7c5815 github.com/tikv/pd/client v0.0.0-20221010134149-d50e5fe43f14 github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144 github.com/twmb/murmur3 v1.1.3 diff --git a/go.sum b/go.sum index 7a21dd8775693..7f2b44c5f2cd0 100644 --- a/go.sum +++ b/go.sum @@ -910,8 +910,8 @@ github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3 h1:f+jULpR github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY= github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJfDRtkanvQPiooDH8HvJ2FBh+iKT/OmiQQ= github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU= -github.com/tikv/client-go/v2 v2.0.1-0.20221017092635-91be9c6ce6c0 h1:5KLqhDGLc/mtemdS/odfOP717rn8ttsTj3jzZ8TZn9A= -github.com/tikv/client-go/v2 v2.0.1-0.20221017092635-91be9c6ce6c0/go.mod h1:9hmGJFrWdehClHg0lv2cYgzvCUEhwLZkH67/PHl75tg= +github.com/tikv/client-go/v2 v2.0.1-0.20221026083454-6c9c7c7c5815 h1:s8eJEGI4p/fxFwMBkoJ+4FAEQNQhHR47TZmVW+EEtOE= +github.com/tikv/client-go/v2 v2.0.1-0.20221026083454-6c9c7c7c5815/go.mod h1:9hmGJFrWdehClHg0lv2cYgzvCUEhwLZkH67/PHl75tg= github.com/tikv/pd/client v0.0.0-20221010134149-d50e5fe43f14 h1:REQOR1XraH1fT9BCoNBPZs1CAe+w7VPLU+d+si7DLYo= github.com/tikv/pd/client v0.0.0-20221010134149-d50e5fe43f14/go.mod h1:E/7+Fkqzwsrp4duzJ2gLPqFl6awU7QG+5yFRXaQwimM= github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144 h1:kl4KhGNsJIbDHS9/4U9yQo1UcPQM0kOMJHn29EoH/Ro= diff --git a/tests/realtikvtest/txntest/txn_test.go b/tests/realtikvtest/txntest/txn_test.go index 914d3a4f0ceee..908329d32f618 100644 --- a/tests/realtikvtest/txntest/txn_test.go +++ b/tests/realtikvtest/txntest/txn_test.go @@ -228,3 +228,26 @@ func TestDuplicateErrorMessage(t *testing.T) { tk2.MustExec("insert into t4 values (1, 2)") tk.MustContainErrMsg("update t3 set c = c + 1 where v = 1", "Duplicate entry '1' for key 't3.i1'") } + +func TestAssertionWhenPessimisticLockLost(t *testing.T) { + store := realtikvtest.CreateMockStoreAndSetup(t) + tk1 := testkit.NewTestKit(t, store) + tk2 := testkit.NewTestKit(t, store) + tk1.MustExec("set @@tidb_constraint_check_in_place_pessimistic=0") + tk1.MustExec("set @@tidb_txn_assertion_level=strict") + tk2.MustExec("set @@tidb_constraint_check_in_place_pessimistic=0") + tk2.MustExec("set @@tidb_txn_assertion_level=strict") + tk1.MustExec("use test") + tk2.MustExec("use test") + tk1.MustExec("create table t (id int primary key, val text)") + tk1.MustExec("begin pessimistic") + tk1.MustExec("select * from t where id = 1 for update") + tk2.MustExec("begin pessimistic") + tk2.MustExec("insert into t values (1, 'b')") + tk2.MustExec("insert into t values (2, 'b')") + tk2.MustExec("commit") + tk1.MustExec("select * from t where id = 2 for update") + tk1.MustExec("insert into t values (1, 'a') on duplicate key update val = concat(val, 'a')") + err := tk1.ExecToErr("commit") + require.NotContains(t, err.Error(), "assertion") +} From 37ed0ab3e4c6e490794bcaacb11935a3571be5ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=B1=E5=B2=9A?= <36239017+YuJuncen@users.noreply.github.com> Date: Mon, 31 Oct 2022 13:11:58 +0000 Subject: [PATCH 05/13] tests: added v1 test for iterators (#38765) --- br/pkg/restore/log_client_test.go | 195 ++++-------------------------- 1 file changed, 25 insertions(+), 170 deletions(-) diff --git a/br/pkg/restore/log_client_test.go b/br/pkg/restore/log_client_test.go index d4c8567b63d82..c4c6d46d15b3f 100644 --- a/br/pkg/restore/log_client_test.go +++ b/br/pkg/restore/log_client_test.go @@ -29,6 +29,8 @@ import ( var id uint64 +type metaMaker = func(files ...*backuppb.DataFileInfo) *backuppb.Metadata + func wm(start, end, minBegin uint64) *backuppb.DataFileInfo { i := wr(start, end, minBegin) i.IsMeta = true @@ -155,7 +157,7 @@ func (b *mockMetaBuilder) b(useV2 bool) (*storage.LocalStorage, string) { return s, path } -func TestReadMetaBetweenTS(t *testing.T) { +func testReadMetaBetweenTSWithVersion(t *testing.T, m metaMaker) { log.SetLevel(zapcore.DebugLevel) type Case struct { items []*backuppb.Metadata @@ -251,103 +253,12 @@ func TestReadMetaBetweenTS(t *testing.T) { } } -func TestReadMetaBetweenTSV2(t *testing.T) { - log.SetLevel(zapcore.DebugLevel) - type Case struct { - items []*backuppb.Metadata - startTS uint64 - endTS uint64 - expectedShiftTS uint64 - expected []int - } - - cases := []Case{ - { - items: []*backuppb.Metadata{ - m2(wr(4, 10, 3), wr(5, 13, 5)), - m2(dr(1, 3)), - m2(wr(10, 42, 9), dr(6, 9)), - }, - startTS: 4, - endTS: 5, - expectedShiftTS: 3, - expected: []int{0, 1}, - }, - { - items: []*backuppb.Metadata{ - m2(wr(1, 100, 1), wr(5, 13, 5), dr(1, 101)), - m2(wr(100, 200, 98), dr(100, 200)), - }, - startTS: 50, - endTS: 99, - expectedShiftTS: 1, - expected: []int{0}, - }, - { - items: []*backuppb.Metadata{ - m2(wr(1, 100, 1), wr(5, 13, 5), dr(1, 101)), - m2(wr(100, 200, 98), dr(100, 200)), - m2(wr(200, 300, 200), dr(200, 300)), - }, - startTS: 150, - endTS: 199, - expectedShiftTS: 98, - expected: []int{1, 0}, - }, - { - items: []*backuppb.Metadata{ - m2(wr(1, 100, 1), wr(5, 13, 5)), - m2(wr(101, 200, 101), dr(100, 200)), - m2(wr(200, 300, 200), dr(200, 300)), - }, - startTS: 150, - endTS: 199, - expectedShiftTS: 101, - expected: []int{1}, - }, - } - - run := func(t *testing.T, c Case) { - req := require.New(t) - ctx := context.Background() - loc, temp := (&mockMetaBuilder{ - metas: c.items, - }).b(true) - defer func() { - t.Log("temp dir", temp) - if !t.Failed() { - os.RemoveAll(temp) - } - }() - init := LogFileManagerInit{ - StartTS: c.startTS, - RestoreTS: c.endTS, - Storage: loc, - } - cli, err := CreateLogFileManager(ctx, init) - req.Equal(cli.ShiftTS(), c.expectedShiftTS) - req.NoError(err) - metas, err := cli.readStreamMeta(ctx) - req.NoError(err) - actualStoreIDs := make([]int64, 0, len(metas)) - for _, meta := range metas { - actualStoreIDs = append(actualStoreIDs, meta.StoreId) - } - expectedStoreIDs := make([]int64, 0, len(c.expected)) - for _, meta := range c.expected { - expectedStoreIDs = append(expectedStoreIDs, c.items[meta].StoreId) - } - req.ElementsMatch(actualStoreIDs, expectedStoreIDs) - } - - for i, c := range cases { - t.Run(fmt.Sprintf("case#%d", i), func(t *testing.T) { - run(t, c) - }) - } +func TestReadMetaBetweenTS(t *testing.T) { + t.Run("MetaV1", func(t *testing.T) { testReadMetaBetweenTSWithVersion(t, m) }) + t.Run("MetaV2", func(t *testing.T) { testReadMetaBetweenTSWithVersion(t, m2) }) } -func TestReadFromMetadata(t *testing.T) { +func testReadFromMetadataWithVersion(t *testing.T, m metaMaker) { type Case struct { items []*backuppb.Metadata untilTS uint64 @@ -413,70 +324,9 @@ func TestReadFromMetadata(t *testing.T) { } } -func TestReadFromMetadataV2(t *testing.T) { - type Case struct { - items []*backuppb.Metadata - untilTS uint64 - expected []int - } - - cases := []Case{ - { - items: []*backuppb.Metadata{ - m2(wr(4, 10, 3), wr(5, 13, 5)), - m2(dr(1, 3)), - m2(wr(10, 42, 9), dr(6, 9)), - }, - untilTS: 10, - expected: []int{0, 1, 2}, - }, - { - items: []*backuppb.Metadata{ - m2(wr(1, 100, 1), wr(5, 13, 5), dr(1, 101)), - m2(wr(100, 200, 98), dr(100, 200)), - }, - untilTS: 99, - expected: []int{0}, - }, - } - - run := func(t *testing.T, c Case) { - req := require.New(t) - ctx := context.Background() - loc, temp := (&mockMetaBuilder{ - metas: c.items, - }).b(true) - defer func() { - t.Log("temp dir", temp) - if !t.Failed() { - os.RemoveAll(temp) - } - }() - - meta := new(StreamMetadataSet) - meta.Helper = stream.NewMetadataHelper() - meta.LoadUntil(ctx, loc, c.untilTS) - - var metas []*backuppb.Metadata - for _, m := range meta.metadata { - metas = append(metas, m) - } - actualStoreIDs := make([]int64, 0, len(metas)) - for _, meta := range metas { - actualStoreIDs = append(actualStoreIDs, meta.StoreId) - } - expectedStoreIDs := make([]int64, 0, len(c.expected)) - for _, meta := range c.expected { - expectedStoreIDs = append(expectedStoreIDs, c.items[meta].StoreId) - } - req.ElementsMatch(actualStoreIDs, expectedStoreIDs) - } - - for i, c := range cases { - t.Run(fmt.Sprintf("case#%d", i), func(t *testing.T) { - run(t, c) - }) - } +func TestReadFromMetadata(t *testing.T) { + t.Run("MetaV1", func(t *testing.T) { testReadFromMetadataWithVersion(t, m) }) + t.Run("MetaV2", func(t *testing.T) { testReadFromMetadataWithVersion(t, m2) }) } func dataFileInfoMatches(t *testing.T, listA []*backuppb.DataFileInfo, listB ...*backuppb.DataFileInfo) { @@ -528,7 +378,7 @@ func formatL(l []*backuppb.DataFileInfo) string { return "[" + strings.Join(r.Item, ", ") + "]" } -func TestFileManager(t *testing.T) { +func testFileManagerWithMeta(t *testing.T, m metaMaker) { type Case struct { Metadata []*backuppb.Metadata StartTS int @@ -544,9 +394,9 @@ func TestFileManager(t *testing.T) { cases := []Case{ { Metadata: []*backuppb.Metadata{ - m2(wm(5, 10, 1), dm(1, 8), dr(2, 6), wr(4, 5, 2)), - m2(wr(50, 54, 42), dr(42, 50), wr(70, 78, 0)), - m2(dr(100, 101), wr(102, 104, 100)), + m(wm(5, 10, 1), dm(1, 8), dr(2, 6), wr(4, 5, 2)), + m(wr(50, 54, 42), dr(42, 50), wr(70, 78, 0)), + m(dr(100, 101), wr(102, 104, 100)), }, StartTS: 2, RestoreTS: 60, @@ -556,9 +406,9 @@ func TestFileManager(t *testing.T) { }, { Metadata: []*backuppb.Metadata{ - m2(wm(4, 10, 1), dm(1, 8), dr(2, 6), wr(4, 5, 2)), - m2(wr(50, 54, 42), dr(42, 50), wr(70, 78, 0), wm(80, 81, 0), wm(90, 92, 0)), - m2(dr(100, 101), wr(102, 104, 100)), + m(wm(4, 10, 1), dm(1, 8), dr(2, 6), wr(4, 5, 2)), + m(wr(50, 54, 42), dr(42, 50), wr(70, 78, 0), wm(80, 81, 0), wm(90, 92, 0)), + m(dr(100, 101), wr(102, 104, 100)), }, StartTS: 5, RestoreTS: 80, @@ -570,9 +420,9 @@ func TestFileManager(t *testing.T) { }, { Metadata: []*backuppb.Metadata{ - m2(wm(5, 10, 1), dm(1, 8), dr(2, 6), wr(4, 5, 2)), - m2(wr(50, 54, 42), dr(42, 50), wr(70, 78, 0), wm(80, 81, 0), wm(90, 92, 0)), - m2(dr(100, 101), wr(102, 104, 100)), + m(wm(5, 10, 1), dm(1, 8), dr(2, 6), wr(4, 5, 2)), + m(wr(50, 54, 42), dr(42, 50), wr(70, 78, 0), wm(80, 81, 0), wm(90, 92, 0)), + m(dr(100, 101), wr(102, 104, 100)), }, StartTS: 6, RestoreTS: 80, @@ -629,3 +479,8 @@ func TestFileManager(t *testing.T) { t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { run(t, c) }) } } + +func TestFileManger(t *testing.T) { + t.Run("MetaV1", func(t *testing.T) { testFileManagerWithMeta(t, m) }) + t.Run("MetaV2", func(t *testing.T) { testFileManagerWithMeta(t, m2) }) +} From 5a8fd6da574ae016fa1ec6b5a237d99c8a2380cc Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 1 Nov 2022 11:47:59 +0800 Subject: [PATCH 06/13] *: fix goleak when to use CreateMockStoreAndDomain (#38734) close pingcap/tidb#38731 --- testkit/mockstore.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testkit/mockstore.go b/testkit/mockstore.go index 6c22f51b1cd44..21b12eaecd611 100644 --- a/testkit/mockstore.go +++ b/testkit/mockstore.go @@ -69,6 +69,10 @@ func CreateMockStoreAndDomain(t testing.TB, opts ...mockstore.MockTiKVStoreOptio dom := bootstrap(t, store, 500*time.Millisecond) sm := MockSessionManager{} dom.InfoSyncer().SetSessionManager(&sm) + t.Cleanup(func() { + view.Stop() + gctuner.GlobalMemoryLimitTuner.Stop() + }) return schematracker.UnwrapStorage(store), dom } From 08aa32e8886e2914d718267d3eac3c35e467ad5b Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 1 Nov 2022 12:15:59 +0800 Subject: [PATCH 07/13] *: update bazel config (#38745) --- br/pkg/restore/BUILD.bazel | 3 +++ br/pkg/storage/BUILD.bazel | 2 ++ br/pkg/utils/iter/BUILD.bazel | 30 +++++++++++++++++++++ ddl/BUILD.bazel | 3 ++- executor/BUILD.bazel | 1 + executor/seqtest/BUILD.bazel | 1 - infoschema/BUILD.bazel | 2 ++ tests/realtikvtest/addindextest/BUILD.bazel | 2 +- util/memory/BUILD.bazel | 1 + util/servermemorylimit/BUILD.bazel | 18 ++++++++++++- 10 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 br/pkg/utils/iter/BUILD.bazel diff --git a/br/pkg/restore/BUILD.bazel b/br/pkg/restore/BUILD.bazel index 3989c1d97c803..5758b2790781d 100644 --- a/br/pkg/restore/BUILD.bazel +++ b/br/pkg/restore/BUILD.bazel @@ -9,6 +9,7 @@ go_library( "db.go", "import.go", "import_retry.go", + "log_client.go", "merge.go", "pipeline_items.go", "range.go", @@ -41,6 +42,7 @@ go_library( "//br/pkg/stream", "//br/pkg/summary", "//br/pkg/utils", + "//br/pkg/utils/iter", "//config", "//ddl", "//ddl/util", @@ -131,6 +133,7 @@ go_test( "//br/pkg/storage", "//br/pkg/stream", "//br/pkg/utils", + "//br/pkg/utils/iter", "//infoschema", "//kv", "//meta/autoid", diff --git a/br/pkg/storage/BUILD.bazel b/br/pkg/storage/BUILD.bazel index e7773cb35c149..c67a17713b2ca 100644 --- a/br/pkg/storage/BUILD.bazel +++ b/br/pkg/storage/BUILD.bazel @@ -37,7 +37,9 @@ go_library( "@com_github_aws_aws_sdk_go//service/s3/s3manager", "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity", "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//:azblob", + "@com_github_golang_snappy//:snappy", "@com_github_google_uuid//:uuid", + "@com_github_klauspost_compress//zstd", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_kvproto//pkg/brpb", "@com_github_pingcap_log//:log", diff --git a/br/pkg/utils/iter/BUILD.bazel b/br/pkg/utils/iter/BUILD.bazel new file mode 100644 index 0000000000000..0e4c55ed67d56 --- /dev/null +++ b/br/pkg/utils/iter/BUILD.bazel @@ -0,0 +1,30 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "iter", + srcs = [ + "combinator_types.go", + "combinators.go", + "iter.go", + "source.go", + "source_types.go", + ], + importpath = "github.com/pingcap/tidb/br/pkg/utils/iter", + visibility = ["//visibility:public"], + deps = [ + "//br/pkg/utils", + "@org_golang_x_exp//constraints", + "@org_golang_x_sync//errgroup", + ], +) + +go_test( + name = "iter_test", + srcs = ["combinator_test.go"], + flaky = True, + race = "on", + deps = [ + ":iter", + "@com_github_stretchr_testify//require", + ], +) diff --git a/ddl/BUILD.bazel b/ddl/BUILD.bazel index 3ccb33749d004..803b066378a56 100644 --- a/ddl/BUILD.bazel +++ b/ddl/BUILD.bazel @@ -219,6 +219,7 @@ go_test( "//parser/terror", "//parser/types", "//planner/core", + "//server", "//session", "//sessionctx", "//sessionctx/stmtctx", @@ -256,7 +257,7 @@ go_test( "@com_github_tikv_client_go_v2//testutils", "@com_github_tikv_client_go_v2//tikv", "@io_etcd_go_etcd_client_v3//:client", - "@io_etcd_go_etcd_tests_v3//integration", + "@org_golang_google_grpc//:grpc", "@org_golang_x_exp//slices", "@org_uber_go_atomic//:atomic", "@org_uber_go_goleak//:goleak", diff --git a/executor/BUILD.bazel b/executor/BUILD.bazel index d4fdc7b5aee3b..c7957510b1297 100644 --- a/executor/BUILD.bazel +++ b/executor/BUILD.bazel @@ -183,6 +183,7 @@ go_library( "//util/rowDecoder", "//util/rowcodec", "//util/sem", + "//util/servermemorylimit", "//util/set", "//util/size", "//util/sqlexec", diff --git a/executor/seqtest/BUILD.bazel b/executor/seqtest/BUILD.bazel index 5de83a3501269..fa66357c81cc7 100644 --- a/executor/seqtest/BUILD.bazel +++ b/executor/seqtest/BUILD.bazel @@ -23,7 +23,6 @@ go_test( "//parser/ast", "//parser/model", "//parser/mysql", - "//parser/terror", "//planner/core", "//server", "//session", diff --git a/infoschema/BUILD.bazel b/infoschema/BUILD.bazel index fb8abc3b33938..53bf35a2d43e3 100644 --- a/infoschema/BUILD.bazel +++ b/infoschema/BUILD.bazel @@ -99,6 +99,8 @@ go_test( "//testkit/testutil", "//types", "//util", + "//util/gctuner", + "//util/memory", "//util/pdapi", "//util/resourcegrouptag", "//util/set", diff --git a/tests/realtikvtest/addindextest/BUILD.bazel b/tests/realtikvtest/addindextest/BUILD.bazel index 99420e080b846..1a9951fd86a51 100644 --- a/tests/realtikvtest/addindextest/BUILD.bazel +++ b/tests/realtikvtest/addindextest/BUILD.bazel @@ -26,8 +26,8 @@ go_test( "add_index_test.go", "concurrent_ddl_test.go", "failpoints_test.go", + "integration_test.go", "main_test.go", - "memory_test.go", "multi_schema_change_test.go", "pitr_test.go", ], diff --git a/util/memory/BUILD.bazel b/util/memory/BUILD.bazel index 17c7a048536a3..c9c443a08f281 100644 --- a/util/memory/BUILD.bazel +++ b/util/memory/BUILD.bazel @@ -17,6 +17,7 @@ go_library( "//util/cgroup", "//util/dbterror", "//util/logutil", + "//util/mathutil", "@com_github_shirou_gopsutil_v3//mem", "@org_golang_x_exp//slices", "@org_uber_go_atomic//:atomic", diff --git a/util/servermemorylimit/BUILD.bazel b/util/servermemorylimit/BUILD.bazel index b03c059049567..9e144b820922e 100644 --- a/util/servermemorylimit/BUILD.bazel +++ b/util/servermemorylimit/BUILD.bazel @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "servermemorylimit", @@ -6,7 +6,23 @@ go_library( importpath = "github.com/pingcap/tidb/util/servermemorylimit", visibility = ["//visibility:public"], deps = [ + "//parser/mysql", + "//types", "//util", "//util/memory", + "@org_uber_go_atomic//:atomic", + ], +) + +go_test( + name = "servermemorylimit_test", + srcs = ["servermemorylimit_test.go"], + embed = [":servermemorylimit"], + flaky = True, + race = "on", + deps = [ + "//types", + "//util", + "@com_github_stretchr_testify//require", ], ) From 38e9aa02ef2ca6286bb5eead5075fb1e0e79aebc Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 1 Nov 2022 13:07:59 +0800 Subject: [PATCH 08/13] *: support foreign key on update cascade and set null when execute update statement (#38652) close pingcap/tidb#38651 --- executor/adapter.go | 2 +- executor/builder.go | 4 + executor/fktest/foreign_key_test.go | 360 ++++++++++++++++++++++++++- executor/foreign_key.go | 133 ++++++++-- executor/insert.go | 2 +- executor/insert_common.go | 3 +- executor/update.go | 13 +- executor/write.go | 8 +- planner/core/common_plans.go | 3 +- planner/core/foreign_key.go | 49 ++-- planner/core/logical_plan_builder.go | 2 +- planner/core/planbuilder.go | 2 +- planner/core/point_get_plan.go | 2 +- 13 files changed, 535 insertions(+), 48 deletions(-) diff --git a/executor/adapter.go b/executor/adapter.go index 9466ba4f80144..fc4b69c929be6 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -612,7 +612,7 @@ func (a *ExecStmt) handleForeignKeyTrigger(ctx context.Context, e Executor, dept } func (a *ExecStmt) handleForeignKeyCascade(ctx context.Context, fkc *FKCascadeExec, depth int) error { - if len(fkc.fkValues) == 0 { + if len(fkc.fkValues) == 0 && len(fkc.fkUpdatedValuesMap) == 0 { return nil } if depth > maxForeignKeyCascadeDepth { diff --git a/executor/builder.go b/executor/builder.go index b08bf123b3c52..4df7a5f123a1d 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -2257,6 +2257,10 @@ func (b *executorBuilder) buildUpdate(v *plannercore.Update) Executor { if b.err != nil { return nil } + updateExec.fkCascades, b.err = b.buildTblID2FKCascadeExecs(tblID2table, v.FKCascades) + if b.err != nil { + return nil + } return updateExec } diff --git a/executor/fktest/foreign_key_test.go b/executor/fktest/foreign_key_test.go index fe977e2dd506b..ac20a0c43c9df 100644 --- a/executor/fktest/foreign_key_test.go +++ b/executor/fktest/foreign_key_test.go @@ -23,6 +23,7 @@ import ( "time" "github.com/pingcap/tidb/executor" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/parser/model" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/testkit" @@ -1264,11 +1265,26 @@ func TestForeignKeyGenerateCascadeSQL(t *testing.T) { sql, err = executor.GenCascadeSetNullSQL(model.NewCIStr("test"), model.NewCIStr("t"), model.NewCIStr(""), fk, fkValues) require.NoError(t, err) - require.Equal(t, "UPDATE `test`.`t` SET `c0`=NULL, `c1`=NULL WHERE (`c0`, `c1`) IN ((1,'a'), (2,'b'))", sql) + require.Equal(t, "UPDATE `test`.`t` SET `c0` = NULL, `c1` = NULL WHERE (`c0`, `c1`) IN ((1,'a'), (2,'b'))", sql) sql, err = executor.GenCascadeSetNullSQL(model.NewCIStr("test"), model.NewCIStr("t"), model.NewCIStr("idx"), fk, fkValues) require.NoError(t, err) - require.Equal(t, "UPDATE `test`.`t` USE INDEX(`idx`) SET `c0`=NULL, `c1`=NULL WHERE (`c0`, `c1`) IN ((1,'a'), (2,'b'))", sql) + require.Equal(t, "UPDATE `test`.`t` USE INDEX(`idx`) SET `c0` = NULL, `c1` = NULL WHERE (`c0`, `c1`) IN ((1,'a'), (2,'b'))", sql) + + newValue1 := []types.Datum{types.NewDatum(10), types.NewDatum("aa")} + couple := &executor.UpdatedValuesCouple{ + NewValues: newValue1, + OldValuesList: fkValues, + } + sql, err = executor.GenCascadeUpdateSQL(model.NewCIStr("test"), model.NewCIStr("t"), model.NewCIStr(""), fk, couple) + require.NoError(t, err) + require.Equal(t, "UPDATE `test`.`t` SET `c0` = 10, `c1` = 'aa' WHERE (`c0`, `c1`) IN ((1,'a'), (2,'b'))", sql) + + newValue2 := []types.Datum{types.NewDatum(nil), types.NewDatum(nil)} + couple.NewValues = newValue2 + sql, err = executor.GenCascadeUpdateSQL(model.NewCIStr("test"), model.NewCIStr("t"), model.NewCIStr("idx"), fk, couple) + require.NoError(t, err) + require.Equal(t, "UPDATE `test`.`t` USE INDEX(`idx`) SET `c0` = NULL, `c1` = NULL WHERE (`c0`, `c1`) IN ((1,'a'), (2,'b'))", sql) } func TestForeignKeyOnDeleteSetNull(t *testing.T) { @@ -1585,6 +1601,330 @@ func TestForeignKeyOnDeleteSetNull2(t *testing.T) { tk.MustQuery("select count(*) from t2 where id is null").Check(testkit.Rows("32768")) } +func TestForeignKeyOnUpdateCascade(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("set @@global.tidb_enable_foreign_key=1") + tk.MustExec("set @@foreign_key_checks=1") + tk.MustExec("use test") + + cases := []struct { + prepareSQLs []string + }{ + // Case-1: test unique index only contain foreign key columns. + { + prepareSQLs: []string{ + "create table t1 (id int, a int, b int, unique index(a, b));", + "create table t2 (b int, name varchar(10), a int, id int, unique index (a,b), foreign key fk(a, b) references t1(a, b) ON UPDATE CASCADE);", + }, + }, + // Case-2: test unique index contain foreign key columns and other columns. + { + prepareSQLs: []string{ + "create table t1 (id int key, a int, b int, unique index(a, b, id));", + "create table t2 (b int, name varchar(10), a int, id int key, unique index (a,b, id), foreign key fk(a, b) references t1(a, b) ON UPDATE CASCADE);", + }, + }, + // Case-3: test non-unique index only contain foreign key columns. + { + prepareSQLs: []string{ + "create table t1 (id int key,a int, b int, index(a, b));", + "create table t2 (b int, a int, name varchar(10), id int key, index (a, b), foreign key fk(a, b) references t1(a, b) ON UPDATE CASCADE);", + }, + }, + // Case-4: test non-unique index contain foreign key columns and other columns. + { + prepareSQLs: []string{ + "create table t1 (id int key,a int, b int, index(a, b, id));", + "create table t2 (name varchar(10), b int, id int key, a int, index (a, b, id), foreign key fk(a, b) references t1(a, b) ON UPDATE CASCADE);", + }, + }, + } + + for idx, ca := range cases { + tk.MustExec("drop table if exists t2;") + tk.MustExec("drop table if exists t1;") + for _, sql := range ca.prepareSQLs { + tk.MustExec(sql) + } + tk.MustExec("insert into t1 (id, a, b) values (1, 11, 21),(2, 12, 22), (3, 13, 23), (4, 14, 24), (5, 15, null), (6, null, 26), (7, null, null);") + tk.MustExec("insert into t2 (id, a, b, name) values (1, 11, 21, 'a'),(2, 12, 22, 'b'), (3, 13, 23, 'c'), (4, 14, 24, 'd'), (5, 15, null, 'e'), (6, null, 26, 'f'), (7, null, null, 'g');") + tk.MustExec("update t1 set a=a+100, b = b+200 where id in (1, 2)") + tk.MustQuery("select id, a, b from t1 where id in (1,2) order by id").Check(testkit.Rows("1 111 221", "2 112 222")) + tk.MustQuery("select id, a, b, name from t2 where id in (1,2,3) order by id").Check(testkit.Rows("1 111 221 a", "2 112 222 b", "3 13 23 c")) + // Test update fk column to null + tk.MustExec("update t1 set a=101, b=null where id = 1 or b = 222") + tk.MustQuery("select id, a, b from t1 where id in (1,2) order by id").Check(testkit.Rows("1 101 ", "2 101 ")) + tk.MustQuery("select id, a, b, name from t2 where id in (1,2,3) order by id").Check(testkit.Rows("1 101 a", "2 101 b", "3 13 23 c")) + tk.MustExec("update t1 set a=null where b is null") + tk.MustQuery("select id, a, b from t1 where b is null order by id").Check(testkit.Rows("1 ", "2 ", "5 ", "7 ")) + tk.MustQuery("select id, a, b, name from t2 where b is null order by id").Check(testkit.Rows("1 101 a", "2 101 b", "5 15 e", "7 g")) + // Test update fk column from null to not-null value + tk.MustExec("update t1 set a=0, b = 0 where id = 7") + tk.MustQuery("select id, a, b from t1 where a=0 and b=0 order by id").Check(testkit.Rows("7 0 0")) + tk.MustQuery("select id, a, b from t2 where a=0 and b=0 order by id").Check(testkit.Rows()) + + // Test in transaction. + tk.MustExec("delete from t2") + tk.MustExec("delete from t1") + tk.MustExec("begin") + tk.MustExec("insert into t1 values (1, 1, 1),(2, 2, 2), (3, 3, 3), (4, 4, 4), (5, 5, null), (6, null, 6), (7, null, null);") + tk.MustExec("insert into t2 (id, a, b, name) values (1, 1, 1, 'a'),(2, 2, 2, 'b'), (3, 3, 3, 'c'), (4, 4, 4, 'd'), (5, 5, null, 'e'), (6, null, 6, 'f'), (7, null, null, 'g');") + tk.MustExec("update t1 set a=a+100, b = b+200 where id in (1, 2)") + tk.MustQuery("select id, a, b, name from t2 order by id").Check(testkit.Rows("1 101 201 a", "2 102 202 b", "3 3 3 c", "4 4 4 d", "5 5 e", "6 6 f", "7 g")) + tk.MustExec("rollback") + tk.MustQuery("select * from t1").Check(testkit.Rows()) + tk.MustQuery("select * from t2").Check(testkit.Rows()) + + tk.MustExec("insert into t1 values (1, 1, 1),(2, 2, 2);") + tk.MustExec("begin") + tk.MustExec("insert into t2 (id, a, b, name) values (1, 1, 1, 'a'),(2, 2, 2, 'b')") + tk.MustExec("update t1 set a=101 where a = 1") + tk.MustQuery("select id, a, b from t1 order by id").Check(testkit.Rows("1 101 1", "2 2 2")) + tk.MustQuery("select id, a, b, name from t2 order by id").Check(testkit.Rows("1 101 1 a", "2 2 2 b")) + err := tk.ExecToErr("insert into t2 (id, a, b, name) values (3, 1, 1, 'c')") + require.Error(t, err) + require.True(t, plannercore.ErrNoReferencedRow2.Equal(err), err.Error()) + tk.MustExec("insert into t1 values (3, 1, 1);") + tk.MustExec("insert into t2 (id, a, b, name) values (3, 1, 1, 'c')") + tk.MustQuery("select id, a, b from t1 order by id").Check(testkit.Rows("1 101 1", "2 2 2", "3 1 1")) + tk.MustQuery("select id, a, b, name from t2 order by id, a").Check(testkit.Rows("1 101 1 a", "2 2 2 b", "3 1 1 c")) + tk.MustExec("update t1 set a=null, b=2000 where id in (1, 2)") + tk.MustExec("commit") + tk.MustQuery("select id, a, b from t1 order by id").Check(testkit.Rows("1 2000", "2 2000", "3 1 1")) + tk.MustQuery("select id, a, b, name from t2 order by id").Check(testkit.Rows("1 2000 a", "2 2000 b", "3 1 1 c")) + + // only test in non-unique index + if idx >= 2 { + tk.MustExec("delete from t2") + tk.MustExec("delete from t1") + tk.MustExec("insert into t1 values (1, 1, 1),(2, 1, 1);") + tk.MustExec("begin") + tk.MustExec("update t1 set a=101 where id = 1") + tk.MustExec("insert into t2 (id, a, b, name) values (1, 1, 1, 'a')") + tk.MustExec("update t1 set b=102 where id = 2") + tk.MustQuery("select * from t1").Check(testkit.Rows("1 101 1", "2 1 102")) + tk.MustQuery("select id, a, b, name from t2").Check(testkit.Rows("1 1 102 a")) + err := tk.ExecToErr("insert into t2 (id, a, b, name) values (3, 1, 1, 'e')") + require.Error(t, err) + require.True(t, plannercore.ErrNoReferencedRow2.Equal(err), err.Error()) + tk.MustExec("insert into t1 values (3, 1, 1);") + tk.MustExec("insert into t2 (id, a, b, name) values (3, 1, 1, 'e')") + tk.MustExec("commit") + tk.MustQuery("select id, a, b from t1 order by id").Check(testkit.Rows("1 101 1", "2 1 102", "3 1 1")) + tk.MustQuery("select id, a, b, name from t2 order by id").Check(testkit.Rows("1 1 102 a", "3 1 1 e")) + + tk.MustExec("delete from t2") + tk.MustExec("delete from t1") + tk.MustExec("begin") + tk.MustExec("insert into t1 values (1, 1, 1),(2, 1, 1);") + tk.MustExec("insert into t2 (id, a, b, name) values (1, 1, 1, 'a'), (2, 1, 1, 'b')") + tk.MustExec("update t1 set a=101, b=102 where id = 1") + tk.MustExec("commit") + tk.MustQuery("select id, a, b from t1 order by id").Check(testkit.Rows("1 101 102", "2 1 1")) + tk.MustQuery("select id, a, b, name from t2 order by id").Check(testkit.Rows("1 101 102 a", "2 101 102 b")) + } + } + + cases = []struct { + prepareSQLs []string + }{ + // Case-5: test primary key only contain foreign key columns, and disable tidb_enable_clustered_index. + { + prepareSQLs: []string{ + "set @@tidb_enable_clustered_index=0;", + "create table t1 (id int, a int, b int, primary key (a, b));", + "create table t2 (b int, a int, name varchar(10), id int, primary key (a, b), foreign key fk(a, b) references t1(a, b) ON UPDATE CASCADE);", + }, + }, + // Case-6: test primary key only contain foreign key columns, and enable tidb_enable_clustered_index. + { + prepareSQLs: []string{ + "set @@tidb_enable_clustered_index=1;", + "create table t1 (id int, a int, b int, primary key (a, b));", + "create table t2 (name varchar(10), b int, a int, id int, primary key (a, b), foreign key fk(a, b) references t1(a, b) ON UPDATE CASCADE);", + }, + }, + // Case-7: test primary key contain foreign key columns and other column, and disable tidb_enable_clustered_index. + { + prepareSQLs: []string{ + "set @@tidb_enable_clustered_index=0;", + "create table t1 (id int, a int, b int, primary key (a, b, id));", + "create table t2 (b int, name varchar(10), a int, id int, primary key (a, b, id), foreign key fk(a, b) references t1(a, b) ON UPDATE CASCADE);", + }, + }, + // Case-8: test primary key contain foreign key columns and other column, and enable tidb_enable_clustered_index. + { + prepareSQLs: []string{ + "set @@tidb_enable_clustered_index=1;", + "create table t1 (id int, a int, b int, primary key (a, b, id));", + "create table t2 (b int, a int, id int, name varchar(10), primary key (a, b, id), foreign key fk(a, b) references t1(a, b) ON UPDATE CASCADE);", + }, + }, + } + for idx, ca := range cases { + tk.MustExec("drop table if exists t2;") + tk.MustExec("drop table if exists t1;") + for _, sql := range ca.prepareSQLs { + tk.MustExec(sql) + } + tk.MustExec("insert into t1 (id, a, b) values (1, 11, 21),(2, 12, 22), (3, 13, 23), (4, 14, 24)") + tk.MustExec("insert into t2 (id, a, b, name) values (1, 11, 21, 'a'),(2, 12, 22, 'b'), (3, 13, 23, 'c'), (4, 14, 24, 'd')") + tk.MustExec("update t1 set a=a+100, b = b+200 where id in (1, 2)") + tk.MustQuery("select id, a, b from t1 where id in (1,2) order by id").Check(testkit.Rows("1 111 221", "2 112 222")) + tk.MustQuery("select id, a, b, name from t2 where id in (1,2,3) order by id").Check(testkit.Rows("1 111 221 a", "2 112 222 b", "3 13 23 c")) + tk.MustExec("update t1 set a=101 where id = 1 or b = 222") + tk.MustQuery("select id, a, b from t1 where id in (1,2) order by id").Check(testkit.Rows("1 101 221", "2 101 222")) + tk.MustQuery("select id, a, b, name from t2 where id in (1,2,3) order by id").Check(testkit.Rows("1 101 221 a", "2 101 222 b", "3 13 23 c")) + + if idx < 2 { + tk.MustGetDBError("update t1 set b=200 where id in (1,2);", kv.ErrKeyExists) + } + + // test in transaction. + tk.MustExec("delete from t2") + tk.MustExec("delete from t1") + tk.MustExec("begin") + tk.MustExec("insert into t1 values (1, 1, 1),(2, 2, 2), (3, 3, 3), (4, 4, 4);") + tk.MustExec("insert into t2 (id, a, b, name) values (1, 1, 1, 'a'),(2, 2, 2, 'b'), (3, 3, 3, 'c'), (4, 4, 4, 'd');") + tk.MustExec("update t1 set a=a+100, b=b+200 where id = 1 or a = 2") + tk.MustExec("update t1 set a=a+1000, b=b+2000 where a in (2,3,4) or b in (5,6,7) or id=2") + tk.MustQuery("select id, a, b from t2 order by id").Check(testkit.Rows("1 101 201", "2 1102 2202", "3 1003 2003", "4 1004 2004")) + tk.MustQuery("select id, a, b, name from t2 order by id").Check(testkit.Rows("1 101 201 a", "2 1102 2202 b", "3 1003 2003 c", "4 1004 2004 d")) + tk.MustExec("commit") + tk.MustQuery("select id, a, b from t2 order by id").Check(testkit.Rows("1 101 201", "2 1102 2202", "3 1003 2003", "4 1004 2004")) + tk.MustQuery("select id, a, b, name from t2 order by id").Check(testkit.Rows("1 101 201 a", "2 1102 2202 b", "3 1003 2003 c", "4 1004 2004 d")) + + tk.MustExec("delete from t2") + tk.MustExec("delete from t1") + tk.MustExec("insert into t1 values (1, 1, 1),(2, 2, 2);") + tk.MustExec("begin") + tk.MustExec("insert into t2 (id, a, b, name) values (1, 1, 1, 'a'),(2, 2, 2, 'b')") + tk.MustExec("update t1 set a=a+100, b=b+200 where id = 1") + tk.MustQuery("select id, a, b from t1 order by id").Check(testkit.Rows("1 101 201", "2 2 2")) + tk.MustQuery("select id, a, b, name from t2 order by id").Check(testkit.Rows("1 101 201 a", "2 2 2 b")) + err := tk.ExecToErr("insert into t2 (id, a, b, name) values (3, 1, 1, 'e')") + require.Error(t, err) + require.True(t, plannercore.ErrNoReferencedRow2.Equal(err), err.Error()) + tk.MustExec("insert into t1 values (3, 1, 1);") + tk.MustExec("insert into t2 (id, a, b, name) values (3, 1, 1, 'c')") + tk.MustQuery("select id, a, b from t1 order by id").Check(testkit.Rows("1 101 201", "2 2 2", "3 1 1")) + tk.MustQuery("select id, a, b, name from t2 order by id").Check(testkit.Rows("1 101 201 a", "2 2 2 b", "3 1 1 c")) + tk.MustExec("update t1 set a=a+1000, b=b+2000 where a>1") + tk.MustExec("commit") + tk.MustQuery("select id, a, b from t1 order by id").Check(testkit.Rows("1 1101 2201", "2 1002 2002", "3 1 1")) + tk.MustQuery("select id, a, b, name from t2 order by id").Check(testkit.Rows("1 1101 2201 a", "2 1002 2002 b", "3 1 1 c")) + } + + // Case-9: test primary key is handle and contain foreign key column. + tk.MustExec("drop table if exists t2;") + tk.MustExec("drop table if exists t1;") + tk.MustExec("set @@tidb_enable_clustered_index=0;") + tk.MustExec("create table t1 (id int, a int, b int, primary key (id));") + tk.MustExec("create table t2 (b int, a int, id int, name varchar(10), primary key (a), foreign key fk(a) references t1(id) ON UPDATE CASCADE);") + tk.MustExec("insert into t1 (id, a, b) values (1, 11, 21),(2, 12, 22), (3, 13, 23), (4, 14, 24)") + tk.MustExec("insert into t2 (id, a, b, name) values (11, 1, 21, 'a'),(12, 2, 22, 'b'), (13, 3, 23, 'c'), (14, 4, 24, 'd')") + tk.MustExec("update t1 set id = id + 100 where id in (1, 2, 3)") + tk.MustQuery("select id, a, b from t1 order by id").Check(testkit.Rows("4 14 24", "101 11 21", "102 12 22", "103 13 23")) + tk.MustQuery("select id, a, b, name from t2 order by id").Check(testkit.Rows("11 101 21 a", "12 102 22 b", "13 103 23 c", "14 4 24 d")) +} + +func TestForeignKeyOnUpdateCascade2(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("set @@global.tidb_enable_foreign_key=1") + tk.MustExec("set @@foreign_key_checks=1") + tk.MustExec("use test") + + // Test update same old row in parent, but only the first old row do cascade update + tk.MustExec("create table t1 (id int key, a int, index (a));") + tk.MustExec("create table t2 (id int key, pid int, constraint fk_pid foreign key (pid) references t1(a) ON UPDATE CASCADE);") + tk.MustExec("insert into t1 (id, a) values (1,1), (2, 1)") + tk.MustExec("insert into t2 (id, pid) values (1,1), (2, 1)") + tk.MustExec("update t1 set a=id+1") + tk.MustQuery("select id, a from t1 order by id").Check(testkit.Rows("1 2", "2 3")) + tk.MustQuery("select id, pid from t2 order by id").Check(testkit.Rows("1 2", "2 2")) + + // Test cascade delete in self table. + tk.MustExec("drop table if exists t1, t2") + tk.MustExec("create table t1 (id int key, name varchar(10), leader int, index(leader), foreign key (leader) references t1(id) ON UPDATE CASCADE);") + tk.MustExec("insert into t1 values (1, 'boss', null), (10, 'l1_a', 1), (11, 'l1_b', 1), (12, 'l1_c', 1)") + tk.MustExec("insert into t1 values (100, 'l2_a1', 10)") + tk.MustExec("insert into t1 values (110, 'l2_b1', 11)") + tk.MustExec("insert into t1 values (1000,'l3_a1', 100)") + tk.MustExec("update t1 set id=id+10000 where id=11") + tk.MustQuery("select id, name, leader from t1 order by id").Check(testkit.Rows("1 boss ", "10 l1_a 1", "12 l1_c 1", "100 l2_a1 10", "110 l2_b1 10011", "1000 l3_a1 100", "10011 l1_b 1")) + tk.MustExec("update t1 set id=0 where id=1") + tk.MustQuery("select id, name, leader from t1 order by id").Check(testkit.Rows("0 boss ", "10 l1_a 0", "12 l1_c 0", "100 l2_a1 10", "110 l2_b1 10011", "1000 l3_a1 100", "10011 l1_b 0")) + + // Test explain analyze with foreign key cascade. + tk.MustExec("explain analyze update t1 set id=1 where id=10") + tk.MustQuery("select id, name, leader from t1 order by id").Check(testkit.Rows("0 boss ", "1 l1_a 0", "12 l1_c 0", "100 l2_a1 1", "110 l2_b1 10011", "1000 l3_a1 100", "10011 l1_b 0")) + + // Test cascade delete in self table with string type foreign key. + tk.MustExec("drop table if exists t1, t2") + tk.MustExec("create table t1 (id varchar(100) key, name varchar(10), leader varchar(100), index(leader), foreign key (leader) references t1(id) ON UPDATE CASCADE);") + tk.MustExec("insert into t1 values (1, 'boss', null), (10, 'l1_a', 1), (11, 'l1_b', 1), (12, 'l1_c', 1)") + tk.MustExec("insert into t1 values (100, 'l2_a1', 10)") + tk.MustExec("insert into t1 values (110, 'l2_b1', 11)") + tk.MustExec("insert into t1 values (1000,'l3_a1', 100)") + tk.MustExec("update t1 set id=id+10000 where id=11") + tk.MustQuery("select id, name, leader from t1 order by name").Check(testkit.Rows("1 boss ", "10 l1_a 1", "10011 l1_b 1", "12 l1_c 1", "100 l2_a1 10", "110 l2_b1 10011", "1000 l3_a1 100")) + tk.MustExec("update t1 set id=0 where id=1") + tk.MustQuery("select id, name, leader from t1 order by name").Check(testkit.Rows("0 boss ", "10 l1_a 0", "10011 l1_b 0", "12 l1_c 0", "100 l2_a1 10", "110 l2_b1 10011", "1000 l3_a1 100")) + + // Test cascade delete depth error. + tk.MustExec("drop table if exists t1, t2") + tk.MustExec("create table t0 (id int, unique index(id))") + tk.MustExec("insert into t0 values (1)") + for i := 1; i < 17; i++ { + tk.MustExec(fmt.Sprintf("create table t%v (id int, unique index(id), foreign key (id) references t%v(id) on update cascade)", i, i-1)) + tk.MustExec(fmt.Sprintf("insert into t%v values (1)", i)) + } + tk.MustGetDBError("update t0 set id=10 where id=1;", executor.ErrForeignKeyCascadeDepthExceeded) + tk.MustQuery("select id from t0").Check(testkit.Rows("1")) + tk.MustQuery("select id from t15").Check(testkit.Rows("1")) + tk.MustExec("drop table if exists t16") + tk.MustExec("update t0 set id=10 where id=1;") + tk.MustQuery("select id from t0").Check(testkit.Rows("10")) + tk.MustQuery("select id from t15").Check(testkit.Rows("10")) + for i := 16; i > -1; i-- { + tk.MustExec("drop table if exists t" + strconv.Itoa(i)) + } + + // Test handle many foreign key value in one cascade. + tk.MustExec("create table t1 (id int auto_increment key, b int, index(b));") + tk.MustExec("create table t2 (id int, b int, foreign key fk(b) references t1(b) on update cascade)") + tk.MustExec("insert into t1 (b) values (1),(2),(3),(4),(5),(6),(7),(8);") + for i := 0; i < 12; i++ { + tk.MustExec("insert into t1 (b) select id from t1") + } + tk.MustQuery("select count(*) from t1").Check(testkit.Rows("32768")) + tk.MustExec("insert into t2 select * from t1") + tk.MustExec("update t1 set b=2") + tk.MustQuery("select count(*) from t1 join t2 where t1.id=t2.id and t1.b=t2.b").Check(testkit.Rows("32768")) +} + +func TestForeignKeyOnUpdateSetNull(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("set @@global.tidb_enable_foreign_key=1") + tk.MustExec("set @@foreign_key_checks=1") + tk.MustExec("use test") + + // Test handle many foreign key value in one cascade. + tk.MustExec("create table t1 (id int auto_increment key, b int, index(b));") + tk.MustExec("create table t2 (id int, b int, foreign key fk(b) references t1(b) on update set null)") + tk.MustExec("insert into t1 (b) values (1),(2),(3),(4),(5),(6),(7),(8);") + for i := 0; i < 12; i++ { + tk.MustExec("insert into t1 (b) select id from t1") + } + tk.MustQuery("select count(*) from t1").Check(testkit.Rows("32768")) + tk.MustExec("insert into t2 select * from t1") + tk.MustExec("update t1 set b=b+100000000") + tk.MustQuery("select count(*) from t2 where b is null").Check(testkit.Rows("32768")) +} + func TestShowCreateTableWithForeignKey(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -1611,3 +1951,19 @@ func TestShowCreateTableWithForeignKey(t *testing.T) { " CONSTRAINT `fk2` FOREIGN KEY (`leader2`) REFERENCES `test`.`t1` (`id`)\n" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin")) } + +func TestForeignKeyCascadeOnDiffColumnType(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("set @@global.tidb_enable_foreign_key=1") + tk.MustExec("set @@foreign_key_checks=1") + tk.MustExec("use test") + tk.MustExec("create table t1 (id bit(10), index(id));") + tk.MustExec("create table t2 (id int key, b bit(10), constraint fk foreign key (b) references t1(id) ON DELETE CASCADE ON UPDATE CASCADE);") + tk.MustExec("insert into t1 values (b'01'), (b'10');") + tk.MustExec("insert into t2 values (1, b'01'), (2, b'10');") + tk.MustExec("delete from t1 where id = b'01';") + tk.MustExec("update t1 set id = b'110' where id = b'10';") + tk.MustQuery("select cast(id as unsigned) from t1;").Check(testkit.Rows("6")) + tk.MustQuery("select id, cast(b as unsigned) from t2;").Check(testkit.Rows("2 6")) +} diff --git a/executor/foreign_key.go b/executor/foreign_key.go index 03b724ada59cf..ea45e9bd394b5 100644 --- a/executor/foreign_key.go +++ b/executor/foreign_key.go @@ -65,7 +65,17 @@ type FKCascadeExec struct { referredFK *model.ReferredFKInfo childTable *model.TableInfo fk *model.FKInfo - fkValues [][]types.Datum + // On delete statement, fkValues stores the delete foreign key values. + // On update statement and the foreign key cascade is `SET NULL`, fkValues stores the old foreign key values. + fkValues [][]types.Datum + // new-value-key => UpdatedValuesCouple + fkUpdatedValuesMap map[string]*UpdatedValuesCouple +} + +// UpdatedValuesCouple contains the updated new row the old rows, exporting for test. +type UpdatedValuesCouple struct { + NewValues []types.Datum + OldValuesList [][]types.Datum } func buildTblID2FKCheckExecs(sctx sessionctx.Context, tblID2Table map[int64]table.Table, tblID2FKChecks map[int64][]*plannercore.FKCheck) (map[int64][]*FKCheckExec, error) { @@ -561,12 +571,13 @@ func (b *executorBuilder) buildFKCascadeExec(tbl table.Table, fkCascade *planner fkValuesSet: set.NewStringSet(), } return &FKCascadeExec{ - b: b, - fkValueHelper: helper, - tp: fkCascade.Tp, - referredFK: fkCascade.ReferredFK, - childTable: fkCascade.ChildTable.Meta(), - fk: fkCascade.FK, + b: b, + fkValueHelper: helper, + tp: fkCascade.Tp, + referredFK: fkCascade.ReferredFK, + childTable: fkCascade.ChildTable.Meta(), + fk: fkCascade.FK, + fkUpdatedValuesMap: make(map[string]*UpdatedValuesCouple), }, nil } @@ -579,6 +590,34 @@ func (fkc *FKCascadeExec) onDeleteRow(sc *stmtctx.StatementContext, row []types. return nil } +func (fkc *FKCascadeExec) onUpdateRow(sc *stmtctx.StatementContext, oldRow, newRow []types.Datum) error { + oldVals, err := fkc.fetchFKValuesWithCheck(sc, oldRow) + if err != nil || len(oldVals) == 0 { + return err + } + if model.ReferOptionType(fkc.fk.OnUpdate) == model.ReferOptionSetNull { + fkc.fkValues = append(fkc.fkValues, oldVals) + return nil + } + newVals, err := fkc.fetchFKValues(newRow) + if err != nil { + return err + } + newValsKey, err := codec.EncodeKey(sc, nil, newVals...) + if err != nil { + return err + } + couple := fkc.fkUpdatedValuesMap[string(newValsKey)] + if couple == nil { + couple = &UpdatedValuesCouple{ + NewValues: newVals, + } + } + couple.OldValuesList = append(couple.OldValuesList, oldVals) + fkc.fkUpdatedValuesMap[string(newValsKey)] = couple + return nil +} + func (fkc *FKCascadeExec) buildExecutor(ctx context.Context) (Executor, error) { p, err := fkc.buildFKCascadePlan(ctx) if err != nil || p == nil { @@ -591,17 +630,9 @@ func (fkc *FKCascadeExec) buildExecutor(ctx context.Context) (Executor, error) { var maxHandleFKValueInOneCascade = 1024 func (fkc *FKCascadeExec) buildFKCascadePlan(ctx context.Context) (plannercore.Plan, error) { - if len(fkc.fkValues) == 0 { + if len(fkc.fkValues) == 0 && len(fkc.fkUpdatedValuesMap) == 0 { return nil, nil } - var fkValues [][]types.Datum - if len(fkc.fkValues) <= maxHandleFKValueInOneCascade { - fkValues = fkc.fkValues - fkc.fkValues = nil - } else { - fkValues = fkc.fkValues[:maxHandleFKValueInOneCascade] - fkc.fkValues = fkc.fkValues[maxHandleFKValueInOneCascade:] - } var indexName model.CIStr indexForFK := model.FindIndexByColumns(fkc.childTable, fkc.fk.Cols...) if indexForFK != nil { @@ -611,12 +642,24 @@ func (fkc *FKCascadeExec) buildFKCascadePlan(ctx context.Context) (plannercore.P var err error switch fkc.tp { case plannercore.FKCascadeOnDelete: + fkValues := fkc.fetchOnDeleteOrUpdateFKValues() switch model.ReferOptionType(fkc.fk.OnDelete) { case model.ReferOptionCascade: sqlStr, err = GenCascadeDeleteSQL(fkc.referredFK.ChildSchema, fkc.childTable.Name, indexName, fkc.fk, fkValues) case model.ReferOptionSetNull: sqlStr, err = GenCascadeSetNullSQL(fkc.referredFK.ChildSchema, fkc.childTable.Name, indexName, fkc.fk, fkValues) } + case plannercore.FKCascadeOnUpdate: + switch model.ReferOptionType(fkc.fk.OnUpdate) { + case model.ReferOptionCascade: + couple := fkc.fetchUpdatedValuesCouple() + if couple != nil && len(couple.NewValues) != 0 { + sqlStr, err = GenCascadeUpdateSQL(fkc.referredFK.ChildSchema, fkc.childTable.Name, indexName, fkc.fk, couple) + } + case model.ReferOptionSetNull: + fkValues := fkc.fetchOnDeleteOrUpdateFKValues() + sqlStr, err = GenCascadeSetNullSQL(fkc.referredFK.ChildSchema, fkc.childTable.Name, indexName, fkc.fk, fkValues) + } } if err != nil { return nil, err @@ -646,6 +689,34 @@ func (fkc *FKCascadeExec) buildFKCascadePlan(ctx context.Context) (plannercore.P return finalPlan, err } +func (fkc *FKCascadeExec) fetchOnDeleteOrUpdateFKValues() [][]types.Datum { + var fkValues [][]types.Datum + if len(fkc.fkValues) <= maxHandleFKValueInOneCascade { + fkValues = fkc.fkValues + fkc.fkValues = nil + } else { + fkValues = fkc.fkValues[:maxHandleFKValueInOneCascade] + fkc.fkValues = fkc.fkValues[maxHandleFKValueInOneCascade:] + } + return fkValues +} + +func (fkc *FKCascadeExec) fetchUpdatedValuesCouple() *UpdatedValuesCouple { + for k, couple := range fkc.fkUpdatedValuesMap { + if len(couple.OldValuesList) <= maxHandleFKValueInOneCascade { + delete(fkc.fkUpdatedValuesMap, k) + return couple + } + result := &UpdatedValuesCouple{ + NewValues: couple.NewValues, + OldValuesList: couple.OldValuesList[:maxHandleFKValueInOneCascade], + } + couple.OldValuesList = couple.OldValuesList[maxHandleFKValueInOneCascade:] + return result + } + return nil +} + // GenCascadeDeleteSQL uses to generate cascade delete SQL, export for test. func GenCascadeDeleteSQL(schema, table, idx model.CIStr, fk *model.FKInfo, fkValues [][]types.Datum) (string, error) { buf := bytes.NewBuffer(make([]byte, 0, 48+8*len(fkValues))) @@ -669,6 +740,19 @@ func GenCascadeDeleteSQL(schema, table, idx model.CIStr, fk *model.FKInfo, fkVal // GenCascadeSetNullSQL uses to generate foreign key `SET NULL` SQL, export for test. func GenCascadeSetNullSQL(schema, table, idx model.CIStr, fk *model.FKInfo, fkValues [][]types.Datum) (string, error) { + newValues := make([]types.Datum, len(fk.Cols)) + for i := range fk.Cols { + newValues[i] = types.NewDatum(nil) + } + couple := &UpdatedValuesCouple{ + NewValues: newValues, + OldValuesList: fkValues, + } + return GenCascadeUpdateSQL(schema, table, idx, fk, couple) +} + +// GenCascadeUpdateSQL uses to generate cascade update SQL, export for test. +func GenCascadeUpdateSQL(schema, table, idx model.CIStr, fk *model.FKInfo, couple *UpdatedValuesCouple) (string, error) { buf := bytes.NewBuffer(nil) buf.WriteString("UPDATE `") buf.WriteString(schema.L) @@ -686,10 +770,15 @@ func GenCascadeSetNullSQL(schema, table, idx model.CIStr, fk *model.FKInfo, fkVa if i > 0 { buf.WriteString(", ") } - buf.WriteString("`" + col.L + "`") - buf.WriteString("=NULL") + buf.WriteString("`" + col.L) + buf.WriteString("` = ") + val, err := genFKValueString(couple.NewValues[i]) + if err != nil { + return "", err + } + buf.WriteString(val) } - err := genCascadeSQLWhereCondition(buf, fk, fkValues) + err := genCascadeSQLWhereCondition(buf, fk, couple.OldValuesList) if err != nil { return "", err } @@ -728,6 +817,12 @@ func genCascadeSQLWhereCondition(buf *bytes.Buffer, fk *model.FKInfo, fkValues [ } func genFKValueString(v types.Datum) (string, error) { + switch v.Kind() { + case types.KindNull: + return "NULL", nil + case types.KindMysqlBit: + return v.GetBinaryLiteral().ToBitLiteralString(true), nil + } val, err := v.ToString() if err != nil { return "", err diff --git a/executor/insert.go b/executor/insert.go index 34463c7d780b4..931b8bbf6b480 100644 --- a/executor/insert.go +++ b/executor/insert.go @@ -420,7 +420,7 @@ func (e *InsertExec) doDupRowUpdate(ctx context.Context, handle kv.Handle, oldRo } newData := e.row4Update[:len(oldRow)] - _, err := updateRecord(ctx, e.ctx, handle, oldRow, newData, assignFlag, e.Table, true, e.memTracker, e.fkChecks) + _, err := updateRecord(ctx, e.ctx, handle, oldRow, newData, assignFlag, e.Table, true, e.memTracker, e.fkChecks, e.fkCascades) if err != nil { return err } diff --git a/executor/insert_common.go b/executor/insert_common.go index d708a59770fdd..905a19b2ca015 100644 --- a/executor/insert_common.go +++ b/executor/insert_common.go @@ -98,7 +98,8 @@ type InsertValues struct { isLoadData bool txnInUse sync.Mutex // fkChecks contains the foreign key checkers. - fkChecks []*FKCheckExec + fkChecks []*FKCheckExec + fkCascades []*FKCascadeExec } type defaultVal struct { diff --git a/executor/update.go b/executor/update.go index acfd883a330cf..d9cffabd08355 100644 --- a/executor/update.go +++ b/executor/update.go @@ -68,6 +68,8 @@ type UpdateExec struct { matches []bool // fkChecks contains the foreign key checkers. the map is tableID -> []*FKCheckExec fkChecks map[int64][]*FKCheckExec + // fkCascades contains the foreign key cascade. the map is tableID -> []*FKCascadeExec + fkCascades map[int64][]*FKCascadeExec } // prepare `handles`, `tableUpdatable`, `changed` to avoid re-computations. @@ -194,7 +196,8 @@ func (e *UpdateExec) exec(ctx context.Context, schema *expression.Schema, row, n // Update row fkChecks := e.fkChecks[content.TblID] - changed, err1 := updateRecord(ctx, e.ctx, handle, oldData, newTableData, flags, tbl, false, e.memTracker, fkChecks) + fkCascades := e.fkCascades[content.TblID] + changed, err1 := updateRecord(ctx, e.ctx, handle, oldData, newTableData, flags, tbl, false, e.memTracker, fkChecks, fkCascades) if err1 == nil { _, exist := e.updatedRowKeys[content.Start].Get(handle) memDelta := e.updatedRowKeys[content.Start].Set(handle, changed) @@ -546,10 +549,14 @@ func (e *UpdateExec) GetFKChecks() []*FKCheckExec { // GetFKCascades implements WithForeignKeyTrigger interface. func (e *UpdateExec) GetFKCascades() []*FKCascadeExec { - return nil + fkCascades := make([]*FKCascadeExec, 0, len(e.fkChecks)) + for _, fkc := range e.fkCascades { + fkCascades = append(fkCascades, fkc...) + } + return fkCascades } // HasFKCascades implements WithForeignKeyTrigger interface. func (e *UpdateExec) HasFKCascades() bool { - return false + return len(e.fkCascades) > 0 } diff --git a/executor/write.go b/executor/write.go index 8d4336b07b960..01359b56a2571 100644 --- a/executor/write.go +++ b/executor/write.go @@ -51,7 +51,7 @@ var ( // 1. changed (bool) : does the update really change the row values. e.g. update set i = 1 where i = 1; // 2. err (error) : error in the update. func updateRecord(ctx context.Context, sctx sessionctx.Context, h kv.Handle, oldData, newData []types.Datum, modified []bool, t table.Table, - onDup bool, memTracker *memory.Tracker, fkChecks []*FKCheckExec) (bool, error) { + onDup bool, memTracker *memory.Tracker, fkChecks []*FKCheckExec, fkCascades []*FKCascadeExec) (bool, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("executor.updateRecord", opentracing.ChildOf(span.Context())) defer span1.Finish() @@ -220,6 +220,12 @@ func updateRecord(ctx context.Context, sctx sessionctx.Context, h kv.Handle, old return false, err } } + for _, fkc := range fkCascades { + err := fkc.onUpdateRow(sc, oldData, newData) + if err != nil { + return false, err + } + } if onDup { sc.AddAffectedRows(2) } else { diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index e8fb5b0a486b8..7dd628b88718d 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -429,7 +429,8 @@ type Update struct { tblID2Table map[int64]table.Table - FKChecks map[int64][]*FKCheck + FKChecks map[int64][]*FKCheck + FKCascades map[int64][]*FKCascade } // MemoryUsage return the memory usage of Update diff --git a/planner/core/foreign_key.go b/planner/core/foreign_key.go index 86a49e3d2a0b8..8c5b03384ae6e 100644 --- a/planner/core/foreign_key.go +++ b/planner/core/foreign_key.go @@ -53,6 +53,8 @@ type FKCascadeType int8 const ( // FKCascadeOnDelete indicates in delete statement. FKCascadeOnDelete FKCascadeType = 1 + // FKCascadeOnUpdate indicates in update statement. + FKCascadeOnUpdate FKCascadeType = 2 emptyFkCheckSize = int64(unsafe.Sizeof(FKCheck{})) emptyFkCascadeSize = int64(unsafe.Sizeof(FKCascade{})) @@ -80,17 +82,17 @@ func (f *FKCascade) MemoryUsage() (sum int64) { return } -func (p *Insert) buildOnInsertFKChecks(ctx sessionctx.Context, is infoschema.InfoSchema, dbName string) ([]*FKCheck, error) { +func (p *Insert) buildOnInsertFKChecks(ctx sessionctx.Context, is infoschema.InfoSchema, dbName string) error { if !ctx.GetSessionVars().ForeignKeyChecks { - return nil, nil + return nil } tblInfo := p.Table.Meta() fkChecks := make([]*FKCheck, 0, len(tblInfo.ForeignKeys)) updateCols := p.buildOnDuplicateUpdateColumns() if len(updateCols) > 0 { - referredFKChecks, err := buildOnUpdateReferredFKChecks(is, dbName, tblInfo, updateCols) + referredFKChecks, _, err := buildOnUpdateReferredFKTriggers(is, dbName, tblInfo, updateCols) if err != nil { - return nil, err + return err } if len(referredFKChecks) > 0 { fkChecks = append(fkChecks, referredFKChecks...) @@ -103,13 +105,14 @@ func (p *Insert) buildOnInsertFKChecks(ctx sessionctx.Context, is infoschema.Inf failedErr := ErrNoReferencedRow2.FastGenByArgs(fk.String(dbName, tblInfo.Name.L)) fkCheck, err := buildFKCheckOnModifyChildTable(is, fk, failedErr) if err != nil { - return nil, err + return err } if fkCheck != nil { fkChecks = append(fkChecks, fkCheck) } } - return fkChecks, nil + p.FKChecks = fkChecks + return nil } func (p *Insert) buildOnDuplicateUpdateColumns() map[string]struct{} { @@ -120,12 +123,13 @@ func (p *Insert) buildOnDuplicateUpdateColumns() map[string]struct{} { return m } -func (updt *Update) buildOnUpdateFKChecks(ctx sessionctx.Context, is infoschema.InfoSchema, tblID2table map[int64]table.Table) error { +func (updt *Update) buildOnUpdateFKTriggers(ctx sessionctx.Context, is infoschema.InfoSchema, tblID2table map[int64]table.Table) error { if !ctx.GetSessionVars().ForeignKeyChecks { return nil } tblID2UpdateColumns := updt.buildTbl2UpdateColumns() fkChecks := make(map[int64][]*FKCheck) + fkCascades := make(map[int64][]*FKCascade) for tid, tbl := range tblID2table { tblInfo := tbl.Meta() dbInfo, exist := is.SchemaByTable(tblInfo) @@ -137,13 +141,16 @@ func (updt *Update) buildOnUpdateFKChecks(ctx sessionctx.Context, is infoschema. if len(updateCols) == 0 { continue } - referredFKChecks, err := buildOnUpdateReferredFKChecks(is, dbInfo.Name.L, tblInfo, updateCols) + referredFKChecks, referredFKCascades, err := buildOnUpdateReferredFKTriggers(is, dbInfo.Name.L, tblInfo, updateCols) if err != nil { return err } if len(referredFKChecks) > 0 { fkChecks[tid] = append(fkChecks[tid], referredFKChecks...) } + if len(referredFKCascades) > 0 { + fkCascades[tid] = append(fkCascades[tid], referredFKCascades...) + } childFKChecks, err := buildOnUpdateChildFKChecks(is, dbInfo.Name.L, tblInfo, updateCols) if err != nil { return err @@ -153,6 +160,7 @@ func (updt *Update) buildOnUpdateFKChecks(ctx sessionctx.Context, is infoschema. } } updt.FKChecks = fkChecks + updt.FKCascades = fkCascades return nil } @@ -170,7 +178,7 @@ func (del *Delete) buildOnDeleteFKTriggers(ctx sessionctx.Context, is infoschema } referredFKs := is.GetTableReferredForeignKeys(dbInfo.Name.L, tblInfo.Name.L) for _, referredFK := range referredFKs { - fkCheck, fkCascade, err := buildOnDeleteFKTrigger(is, referredFK) + fkCheck, fkCascade, err := buildOnDeleteOrUpdateFKTrigger(is, referredFK, FKCascadeOnDelete) if err != nil { return err } @@ -187,22 +195,26 @@ func (del *Delete) buildOnDeleteFKTriggers(ctx sessionctx.Context, is infoschema return nil } -func buildOnUpdateReferredFKChecks(is infoschema.InfoSchema, dbName string, tblInfo *model.TableInfo, updateCols map[string]struct{}) ([]*FKCheck, error) { +func buildOnUpdateReferredFKTriggers(is infoschema.InfoSchema, dbName string, tblInfo *model.TableInfo, updateCols map[string]struct{}) ([]*FKCheck, []*FKCascade, error) { referredFKs := is.GetTableReferredForeignKeys(dbName, tblInfo.Name.L) fkChecks := make([]*FKCheck, 0, len(referredFKs)) + fkCascades := make([]*FKCascade, 0, len(referredFKs)) for _, referredFK := range referredFKs { if !isMapContainAnyCols(updateCols, referredFK.Cols...) { continue } - fkCheck, err := buildFKCheckOnModifyReferTable(is, referredFK) + fkCheck, fkCascade, err := buildOnDeleteOrUpdateFKTrigger(is, referredFK, FKCascadeOnUpdate) if err != nil { - return nil, err + return nil, nil, err } if fkCheck != nil { fkChecks = append(fkChecks, fkCheck) } + if fkCascade != nil { + fkCascades = append(fkCascades, fkCascade) + } } - return fkChecks, nil + return fkChecks, fkCascades, nil } func buildOnUpdateChildFKChecks(is infoschema.InfoSchema, dbName string, tblInfo *model.TableInfo, updateCols map[string]struct{}) ([]*FKCheck, error) { @@ -260,7 +272,7 @@ func (updt *Update) buildTbl2UpdateColumns() map[int64]map[string]struct{} { return tblID2UpdateColumns } -func buildOnDeleteFKTrigger(is infoschema.InfoSchema, referredFK *model.ReferredFKInfo) (*FKCheck, *FKCascade, error) { +func buildOnDeleteOrUpdateFKTrigger(is infoschema.InfoSchema, referredFK *model.ReferredFKInfo, tp FKCascadeType) (*FKCheck, *FKCascade, error) { childTable, err := is.TableByName(referredFK.ChildSchema, referredFK.ChildTable) if err != nil { return nil, nil, nil @@ -273,12 +285,17 @@ func buildOnDeleteFKTrigger(is infoschema.InfoSchema, referredFK *model.Referred if fk.State != model.StatePublic { fkReferOption = model.ReferOptionRestrict } else { - fkReferOption = model.ReferOptionType(fk.OnDelete) + switch tp { + case FKCascadeOnDelete: + fkReferOption = model.ReferOptionType(fk.OnDelete) + case FKCascadeOnUpdate: + fkReferOption = model.ReferOptionType(fk.OnUpdate) + } } switch fkReferOption { case model.ReferOptionCascade, model.ReferOptionSetNull: fkCascade := &FKCascade{ - Tp: FKCascadeOnDelete, + Tp: tp, ReferredFK: referredFK, ChildTable: childTable, FK: fk, diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index e29e74919f59f..7e9d6c1a01cf8 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -5416,7 +5416,7 @@ func (b *PlanBuilder) buildUpdate(ctx context.Context, update *ast.UpdateStmt) ( } updt.PartitionedTable = b.partitionedTable updt.tblID2Table = tblID2table - err = updt.buildOnUpdateFKChecks(b.ctx, b.is, tblID2table) + err = updt.buildOnUpdateFKTriggers(b.ctx, b.is, tblID2table) return updt, err } diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index cee0fc324d5b6..8886d5c4aab4e 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -3621,7 +3621,7 @@ func (b *PlanBuilder) buildInsert(ctx context.Context, insert *ast.InsertStmt) ( if err != nil { return nil, err } - insertPlan.FKChecks, err = insertPlan.buildOnInsertFKChecks(b.ctx, b.is, tn.DBInfo.Name.L) + err = insertPlan.buildOnInsertFKChecks(b.ctx, b.is, tn.DBInfo.Name.L) return insertPlan, err } diff --git a/planner/core/point_get_plan.go b/planner/core/point_get_plan.go index 125e716522947..de69438257f4e 100644 --- a/planner/core/point_get_plan.go +++ b/planner/core/point_get_plan.go @@ -1593,7 +1593,7 @@ func buildPointUpdatePlan(ctx sessionctx.Context, pointPlan PhysicalPlan, dbName updatePlan.PartitionedTable = append(updatePlan.PartitionedTable, pt) } } - err := updatePlan.buildOnUpdateFKChecks(ctx, is, updatePlan.tblID2Table) + err := updatePlan.buildOnUpdateFKTriggers(ctx, is, updatePlan.tblID2Table) if err != nil { return nil } From ecdc0f74edceff881d01f55fc2bb14df7baa2e7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E8=B6=85?= Date: Tue, 1 Nov 2022 13:33:59 +0800 Subject: [PATCH 09/13] extension: add error and active roles info to `extension.ConnEventInfo` (#38752) close pingcap/tidb#38493 --- extension/BUILD.bazel | 1 + extension/extensionimpl/bootstrap.go | 2 +- extension/extensions.go | 2 +- extension/session.go | 16 +++++--- server/conn.go | 3 +- server/conn_test.go | 17 ++++++-- server/server.go | 42 ++++++++++++++----- server/tidb_test.go | 60 +++++++++++++++++++++------- 8 files changed, 106 insertions(+), 37 deletions(-) diff --git a/extension/BUILD.bazel b/extension/BUILD.bazel index 0ec88aff5a9f3..a21f6ea2f4130 100644 --- a/extension/BUILD.bazel +++ b/extension/BUILD.bazel @@ -13,6 +13,7 @@ go_library( importpath = "github.com/pingcap/tidb/extension", visibility = ["//visibility:public"], deps = [ + "//parser/auth", "//sessionctx/variable", "//types", "//util/chunk", diff --git a/extension/extensionimpl/bootstrap.go b/extension/extensionimpl/bootstrap.go index 1332bc6f9e8c9..8b6b154f988ab 100644 --- a/extension/extensionimpl/bootstrap.go +++ b/extension/extensionimpl/bootstrap.go @@ -63,7 +63,7 @@ func (c *bootstrapContext) SessionPool() extension.SessionPool { return c.sessionPool } -// Bootstrap bootstrap all extensions +// Bootstrap bootstraps all extensions func Bootstrap(ctx context.Context, do *domain.Domain) error { extensions, err := extension.GetExtensions() if err != nil { diff --git a/extension/extensions.go b/extension/extensions.go index e977d20dc4b4e..68bcffd585a89 100644 --- a/extension/extensions.go +++ b/extension/extensions.go @@ -29,7 +29,7 @@ func (es *Extensions) Manifests() []*Manifest { return manifests } -// Bootstrap bootstrap all extensions +// Bootstrap bootstraps all extensions func (es *Extensions) Bootstrap(ctx BootstrapContext) error { if es == nil { return nil diff --git a/extension/session.go b/extension/session.go index f0217f6da9b53..65fddb753163a 100644 --- a/extension/session.go +++ b/extension/session.go @@ -14,10 +14,17 @@ package extension -import "github.com/pingcap/tidb/sessionctx/variable" +import ( + "github.com/pingcap/tidb/parser/auth" + "github.com/pingcap/tidb/sessionctx/variable" +) // ConnEventInfo is the connection info for the event -type ConnEventInfo variable.ConnectionInfo +type ConnEventInfo struct { + *variable.ConnectionInfo + ActiveRoles []*auth.RoleIdentity + Error error +} // ConnEventTp is the type of the connection event type ConnEventTp uint8 @@ -60,13 +67,12 @@ type SessionExtensions struct { } // OnConnectionEvent will be called when a connection event happens -func (es *SessionExtensions) OnConnectionEvent(tp ConnEventTp, info *variable.ConnectionInfo) { +func (es *SessionExtensions) OnConnectionEvent(tp ConnEventTp, event *ConnEventInfo) { if es == nil { return } - eventInfo := ConnEventInfo(*info) for _, fn := range es.connectionEventFuncs { - fn(tp, &eventInfo) + fn(tp, event) } } diff --git a/server/conn.go b/server/conn.go index c2e3f1db9b299..33d07655e5e72 100644 --- a/server/conn.go +++ b/server/conn.go @@ -2510,8 +2510,7 @@ func (cc *clientConn) handleCommonConnectionReset(ctx context.Context) error { connectionInfo := cc.connectInfo() cc.ctx.GetSessionVars().ConnectionInfo = connectionInfo - cc.extensions.OnConnectionEvent(extension.ConnReset, connectionInfo) - + cc.onExtensionConnEvent(extension.ConnReset, nil) err := plugin.ForeachPlugin(plugin.Audit, func(p *plugin.Plugin) error { authPlugin := plugin.DeclareAuditManifest(p.Manifest) if authPlugin.OnConnectionEvent != nil { diff --git a/server/conn_test.go b/server/conn_test.go index 0f9acc6f183d6..28e023ea8c01b 100644 --- a/server/conn_test.go +++ b/server/conn_test.go @@ -1663,7 +1663,10 @@ func TestExtensionChangeUser(t *testing.T) { outBuffer.Reset() } - expectedConnInfo := extension.ConnEventInfo(*cc.connectInfo()) + expectedConnInfo := extension.ConnEventInfo{ + ConnectionInfo: cc.connectInfo(), + ActiveRoles: []*auth.RoleIdentity{}, + } expectedConnInfo.User = "user1" expectedConnInfo.DB = "db1" @@ -1679,7 +1682,9 @@ func TestExtensionChangeUser(t *testing.T) { }) require.True(t, logged) require.Equal(t, extension.ConnReset, logTp) - require.Equal(t, expectedConnInfo, *logInfo) + require.Equal(t, expectedConnInfo.ActiveRoles, logInfo.ActiveRoles) + require.Equal(t, expectedConnInfo.Error, logInfo.Error) + require.Equal(t, *(expectedConnInfo.ConnectionInfo), *(logInfo.ConnectionInfo)) logged = false logTp = 0 @@ -1697,7 +1702,9 @@ func TestExtensionChangeUser(t *testing.T) { }) require.True(t, logged) require.Equal(t, extension.ConnReset, logTp) - require.Equal(t, expectedConnInfo, *logInfo) + require.Equal(t, expectedConnInfo.ActiveRoles, logInfo.ActiveRoles) + require.Equal(t, expectedConnInfo.Error, logInfo.Error) + require.Equal(t, *(expectedConnInfo.ConnectionInfo), *(logInfo.ConnectionInfo)) logged = false logTp = 0 @@ -1710,5 +1717,7 @@ func TestExtensionChangeUser(t *testing.T) { }) require.True(t, logged) require.Equal(t, extension.ConnReset, logTp) - require.Equal(t, expectedConnInfo, *logInfo) + require.Equal(t, expectedConnInfo.ActiveRoles, logInfo.ActiveRoles) + require.Equal(t, expectedConnInfo.Error, logInfo.Error) + require.Equal(t, *(expectedConnInfo.ConnectionInfo), *(logInfo.ConnectionInfo)) } diff --git a/server/server.go b/server/server.go index 9ed4b963913ec..5efef687ecdb0 100644 --- a/server/server.go +++ b/server/server.go @@ -55,6 +55,7 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/auth" "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/parser/terror" "github.com/pingcap/tidb/planner/core" @@ -510,7 +511,6 @@ func (s *Server) onConn(conn *clientConn) { terror.Log(conn.Close()) return } - connectionInfo := conn.connectInfo() extensions, err := extension.GetExtensions() if err != nil { @@ -522,18 +522,17 @@ func (s *Server) onConn(conn *clientConn) { if sessExtensions := extensions.NewSessionExtensions(); sessExtensions != nil { conn.extensions = sessExtensions - sessExtensions.OnConnectionEvent(extension.ConnConnected, connectionInfo) + conn.onExtensionConnEvent(extension.ConnConnected, nil) defer func() { - sessExtensions.OnConnectionEvent(extension.ConnDisconnected, connectionInfo) + conn.onExtensionConnEvent(extension.ConnDisconnected, nil) }() } ctx := logutil.WithConnID(context.Background(), conn.connectionID) if err := conn.handshake(ctx); err != nil { - connectionInfo = conn.connectInfo() - conn.extensions.OnConnectionEvent(extension.ConnHandshakeRejected, connectionInfo) + conn.onExtensionConnEvent(extension.ConnHandshakeRejected, err) if plugin.IsEnable(plugin.Audit) && conn.getCtx() != nil { - conn.getCtx().GetSessionVars().ConnectionInfo = connectionInfo + conn.getCtx().GetSessionVars().ConnectionInfo = conn.connectInfo() err = plugin.ForeachPlugin(plugin.Audit, func(p *plugin.Plugin) error { authPlugin := plugin.DeclareAuditManifest(p.Manifest) if authPlugin.OnConnectionEvent != nil { @@ -578,9 +577,8 @@ func (s *Server) onConn(conn *clientConn) { metrics.ConnGauge.Set(float64(connections)) sessionVars := conn.ctx.GetSessionVars() - connectionInfo = conn.connectInfo() - sessionVars.ConnectionInfo = connectionInfo - conn.extensions.OnConnectionEvent(extension.ConnHandshakeAccepted, connectionInfo) + sessionVars.ConnectionInfo = conn.connectInfo() + conn.onExtensionConnEvent(extension.ConnHandshakeAccepted, nil) err = plugin.ForeachPlugin(plugin.Audit, func(p *plugin.Plugin) error { authPlugin := plugin.DeclareAuditManifest(p.Manifest) if authPlugin.OnConnectionEvent != nil { @@ -638,6 +636,32 @@ func (cc *clientConn) connectInfo() *variable.ConnectionInfo { return connInfo } +func (cc *clientConn) onExtensionConnEvent(tp extension.ConnEventTp, err error) { + if cc.extensions == nil { + return + } + + var connInfo *variable.ConnectionInfo + var activeRoles []*auth.RoleIdentity + if ctx := cc.getCtx(); ctx != nil { + sessVars := ctx.GetSessionVars() + connInfo = sessVars.ConnectionInfo + activeRoles = sessVars.ActiveRoles + } + + if connInfo == nil { + connInfo = cc.connectInfo() + } + + info := &extension.ConnEventInfo{ + ConnectionInfo: connInfo, + ActiveRoles: activeRoles, + Error: err, + } + + cc.extensions.OnConnectionEvent(tp, info) +} + func (s *Server) checkConnectionCount() error { // When the value of Instance.MaxConnections is 0, the number of connections is unlimited. if int(s.cfg.Instance.MaxConnections) == 0 { diff --git a/server/tidb_test.go b/server/tidb_test.go index 293406fe87538..633dab3cef8af 100644 --- a/server/tidb_test.go +++ b/server/tidb_test.go @@ -47,8 +47,10 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/parser" "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/auth" tmysql "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/sessiontxn" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/store/mockstore/unistore" @@ -2825,28 +2827,50 @@ func TestExtensionConnEvent(t *testing.T) { _ = conn.Close() }() - var conn1, conn2 extension.ConnEventInfo + var expectedConn2 variable.ConnectionInfo logs.check(func() { require.Equal(t, []extension.ConnEventTp{ extension.ConnConnected, extension.ConnHandshakeAccepted, }, logs.types) - conn1 = logs.infos[0] - conn2 = conn1 - conn2.User = "root" - conn2.DB = "test" - + conn1 := logs.infos[0] require.Equal(t, "127.0.0.1", conn1.ClientIP) require.Equal(t, "127.0.0.1", conn1.ServerIP) require.Empty(t, conn1.User) require.Empty(t, conn1.DB) - require.Equal(t, conn2, logs.infos[1]) + require.Equal(t, int(ts.port), conn1.ServerPort) + require.NotEqual(t, conn1.ServerPort, conn1.ClientPort) + require.NotEmpty(t, conn1.ConnectionID) + require.Nil(t, conn1.ActiveRoles) + require.NoError(t, conn1.Error) + + expectedConn2 = *(conn1.ConnectionInfo) + expectedConn2.User = "root" + expectedConn2.DB = "test" + require.Equal(t, []*auth.RoleIdentity{}, logs.infos[1].ActiveRoles) + require.Nil(t, logs.infos[1].Error) + require.Equal(t, expectedConn2, *(logs.infos[1].ConnectionInfo)) }) + + _, err = conn.ExecContext(context.TODO(), "create role r1@'%'") + require.NoError(t, err) + _, err = conn.ExecContext(context.TODO(), "grant r1 TO root") + require.NoError(t, err) + _, err = conn.ExecContext(context.TODO(), "set role all") + require.NoError(t, err) + require.NoError(t, conn.Close()) require.NoError(t, db.Close()) require.NoError(t, logs.waitConnDisconnected()) logs.check(func() { - require.Equal(t, conn2, logs.infos[2]) + require.Equal(t, 3, len(logs.infos)) + require.Equal(t, 1, len(logs.infos[2].ActiveRoles)) + require.Equal(t, auth.RoleIdentity{ + Username: "r1", + Hostname: "%", + }, *logs.infos[2].ActiveRoles[0]) + require.Nil(t, logs.infos[2].Error) + require.Equal(t, expectedConn2, *(logs.infos[2].ConnectionInfo)) }) // test for login failed @@ -2871,16 +2895,22 @@ func TestExtensionConnEvent(t *testing.T) { extension.ConnHandshakeRejected, extension.ConnDisconnected, }, logs.types) - conn1 = logs.infos[0] - conn2 = conn1 - conn2.User = "noexist" - conn2.DB = "test" - + conn1 := logs.infos[0] require.Equal(t, "127.0.0.1", conn1.ClientIP) require.Equal(t, "127.0.0.1", conn1.ServerIP) require.Empty(t, conn1.User) require.Empty(t, conn1.DB) - require.Equal(t, conn2, logs.infos[1]) - require.Equal(t, conn2, logs.infos[2]) + require.Equal(t, int(ts.port), conn1.ServerPort) + require.NotEqual(t, conn1.ServerPort, conn1.ClientPort) + require.NotEmpty(t, conn1.ConnectionID) + require.Nil(t, conn1.ActiveRoles) + require.NoError(t, conn1.Error) + + expectedConn2 = *(conn1.ConnectionInfo) + expectedConn2.User = "noexist" + expectedConn2.DB = "test" + require.Equal(t, []*auth.RoleIdentity{}, logs.infos[1].ActiveRoles) + require.EqualError(t, logs.infos[1].Error, "[server:1045]Access denied for user 'noexist'@'127.0.0.1' (using password: NO)") + require.Equal(t, expectedConn2, *(logs.infos[1].ConnectionInfo)) }) } From 7e4d7598bef4e1f0346646cc5d61a3ce147f64cf Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 1 Nov 2022 14:07:59 +0800 Subject: [PATCH 10/13] bazel: use --remote_download_minimal in ci (#38780) --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f72ea097ff98b..7145f2a67d0d5 100644 --- a/Makefile +++ b/Makefile @@ -411,8 +411,10 @@ bazel_coverage_test: failpoint-enable bazel_ci_prepare bazel_build: bazel_ci_prepare mkdir -p bin - bazel $(BAZEL_GLOBAL_CONFIG) build $(BAZEL_CMD_CONFIG) \ + bazel $(BAZEL_GLOBAL_CONFIG) build --remote_download_minimal $(BAZEL_CMD_CONFIG) \ //... --//build:with_nogo_flag=true + bazel $(BAZEL_GLOBAL_CONFIG) build $(BAZEL_CMD_CONFIG) \ + //cmd/importer:importer //tidb-server:tidb-server //tidb-server:tidb-server-check --//build:with_nogo_flag=true cp bazel-out/k8-fastbuild/bin/tidb-server/tidb-server_/tidb-server ./bin cp bazel-out/k8-fastbuild/bin/cmd/importer/importer_/importer ./bin cp bazel-out/k8-fastbuild/bin/tidb-server/tidb-server-check_/tidb-server-check ./bin From cc2a0388b0ee951c492ae85a074c1b8a2d11da0a Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 1 Nov 2022 14:25:59 +0800 Subject: [PATCH 11/13] sessionctx: fix TiDBGOGCTunerThreshold failing to ParseFloat (#38774) close pingcap/tidb#38778 --- sessionctx/variable/sysvar.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 6e156f3aed68a..0e82a45a3d967 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -738,10 +738,7 @@ var defaultSysVars = []*SysVar{ return strconv.FormatFloat(GOGCTunerThreshold.Load(), 'f', -1, 64), nil }, Validation: func(s *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { - floatValue, err := strconv.ParseFloat(normalizedValue, 64) - if err != nil { - return "", err - } + floatValue := tidbOptFloat64(normalizedValue, DefTiDBGOGCTunerThreshold) globalMemoryLimitTuner := gctuner.GlobalMemoryLimitTuner.GetPercentage() if floatValue < 0 && floatValue > 0.9 { return "", ErrWrongValueForVar.GenWithStackByArgs(TiDBGOGCTunerThreshold, normalizedValue) @@ -752,10 +749,7 @@ var defaultSysVars = []*SysVar{ return strconv.FormatFloat(floatValue, 'f', -1, 64), nil }, SetGlobal: func(_ context.Context, s *SessionVars, val string) error { - factor, err := strconv.ParseFloat(val, 64) - if err != nil { - return err - } + factor := tidbOptFloat64(val, DefTiDBGOGCTunerThreshold) GOGCTunerThreshold.Store(factor) memTotal := memory.ServerMemoryLimit.Load() threshold := float64(memTotal) * factor From be75669f68a2eda401b5ea239a9842fb0369f907 Mon Sep 17 00:00:00 2001 From: okJiang <819421878@qq.com> Date: Tue, 1 Nov 2022 16:22:01 +0800 Subject: [PATCH 12/13] dumpling: add progress to dumpling's status (#38625) close pingcap/tidb#38626 --- dumpling/export/config.go | 6 +- dumpling/export/dump.go | 45 ++++-- dumpling/export/dump_test.go | 3 +- dumpling/export/metrics.go | 5 + dumpling/export/status.go | 33 ++++- dumpling/export/status_test.go | 9 +- dumpling/export/task.go | 4 +- dumpling/export/util.go | 41 ++++++ dumpling/export/util_test.go | 14 ++ dumpling/tests/rows_extreme_int/run.sh | 9 +- go.mod | 67 ++++----- go.sum | 185 ++++++++++++++++--------- 12 files changed, 291 insertions(+), 130 deletions(-) diff --git a/dumpling/export/config.go b/dumpling/export/config.go index 44f1f02ae2e83..62ef45b45a0ae 100644 --- a/dumpling/export/config.go +++ b/dumpling/export/config.go @@ -621,9 +621,9 @@ const ( // DefaultTableFilter is the default exclude table filter. It will exclude all system databases DefaultTableFilter = "!/^(mysql|sys|INFORMATION_SCHEMA|PERFORMANCE_SCHEMA|METRICS_SCHEMA|INSPECTION_SCHEMA)$/.*" - defaultDumpThreads = 128 - defaultDumpGCSafePointTTL = 5 * 60 - defaultEtcdDialTimeOut = 3 * time.Second + defaultTaskChannelCapacity = 128 + defaultDumpGCSafePointTTL = 5 * 60 + defaultEtcdDialTimeOut = 3 * time.Second // LooseCollationCompatible is used in DM, represents a collation setting for best compatibility. LooseCollationCompatible = "loose" diff --git a/dumpling/export/dump.go b/dumpling/export/dump.go index 03b6ba015c0a3..7e5a81e0f3ce1 100644 --- a/dumpling/export/dump.go +++ b/dumpling/export/dump.go @@ -235,15 +235,16 @@ func (d *Dumper) Dump() (dumpErr error) { return rebuildMetaConn(conn, updateMeta) } - chanSize := defaultDumpThreads + chanSize := defaultTaskChannelCapacity failpoint.Inject("SmallDumpChanSize", func() { chanSize = 1 }) - taskChan := make(chan Task, chanSize) - AddGauge(d.metrics.taskChannelCapacity, defaultDumpThreads) + taskIn, taskOut := infiniteChan[Task]() + // todo: refine metrics + AddGauge(d.metrics.taskChannelCapacity, float64(chanSize)) wg, writingCtx := errgroup.WithContext(tctx) writerCtx := tctx.WithContext(writingCtx) - writers, tearDownWriters, err := d.startWriters(writerCtx, wg, taskChan, rebuildConn) + writers, tearDownWriters, err := d.startWriters(writerCtx, wg, taskOut, rebuildConn) if err != nil { return err } @@ -291,13 +292,18 @@ func (d *Dumper) Dump() (dumpErr error) { baseConn := newBaseConn(metaConn, true, rebuildMetaConn) if conf.SQL == "" { - if err = d.dumpDatabases(writerCtx, baseConn, taskChan); err != nil && !errors.ErrorEqual(err, context.Canceled) { + if err = d.dumpDatabases(writerCtx, baseConn, taskIn); err != nil && !errors.ErrorEqual(err, context.Canceled) { return err } } else { - d.dumpSQL(writerCtx, baseConn, taskChan) + d.dumpSQL(writerCtx, baseConn, taskIn) } - close(taskChan) + d.metrics.progressReady.Store(true) + close(taskIn) + failpoint.Inject("EnableLogProgress", func() { + time.Sleep(1 * time.Second) + tctx.L().Debug("progress ready, sleep 1s") + }) _ = baseConn.DBConn.Close() if err := wg.Wait(); err != nil { summary.CollectFailureUnit("dump table data", err) @@ -330,11 +336,16 @@ func (d *Dumper) startWriters(tctx *tcontext.Context, wg *errgroup.Group, taskCh // tctx.L().Debug("finished dumping table data", // zap.String("database", td.Meta.DatabaseName()), // zap.String("table", td.Meta.TableName())) + failpoint.Inject("EnableLogProgress", func() { + time.Sleep(1 * time.Second) + tctx.L().Debug("EnableLogProgress, sleep 1s") + }) } }) writer.setFinishTaskCallBack(func(task Task) { IncGauge(d.metrics.taskChannelCapacity) if td, ok := task.(*TaskTableData); ok { + d.metrics.completedChunks.Add(1) tctx.L().Debug("finish dumping table data task", zap.String("database", td.Meta.DatabaseName()), zap.String("table", td.Meta.TableName()), @@ -451,7 +462,6 @@ func (d *Dumper) dumpDatabases(tctx *tcontext.Context, metaConn *BaseConn, taskC } } } - return nil } @@ -624,6 +634,7 @@ func (d *Dumper) buildConcatTask(tctx *tcontext.Context, conn *BaseConn, meta Ta return } tableDataArr = append(tableDataArr, tableDataInst) + d.metrics.totalChunks.Dec() } for { select { @@ -650,7 +661,7 @@ func (d *Dumper) buildConcatTask(tctx *tcontext.Context, conn *BaseConn, meta Ta return nil, nil } } - return NewTaskTableData(meta, newMultiQueriesChunk(queries, colLen), 0, 1), nil + return d.newTaskTableData(meta, newMultiQueriesChunk(queries, colLen), 0, 1), nil } return nil, err case task := <-tableChan: @@ -662,7 +673,7 @@ func (d *Dumper) buildConcatTask(tctx *tcontext.Context, conn *BaseConn, meta Ta func (d *Dumper) dumpWholeTableDirectly(tctx *tcontext.Context, meta TableMeta, taskChan chan<- Task, partition, orderByClause string, currentChunk, totalChunks int) error { conf := d.conf tableIR := SelectAllFromTable(conf, meta, partition, orderByClause) - task := NewTaskTableData(meta, tableIR, currentChunk, totalChunks) + task := d.newTaskTableData(meta, tableIR, currentChunk, totalChunks) ctxDone := d.sendTaskToChan(tctx, task, taskChan) if ctxDone { return tctx.Err() @@ -775,7 +786,7 @@ func (d *Dumper) concurrentDumpTable(tctx *tcontext.Context, conn *BaseConn, met if len(nullValueCondition) > 0 { nullValueCondition = "" } - task := NewTaskTableData(meta, newTableData(query, selectLen, false), chunkIndex, int(totalChunks)) + task := d.newTaskTableData(meta, newTableData(query, selectLen, false), chunkIndex, int(totalChunks)) ctxDone := d.sendTaskToChan(tctx, task, taskChan) if ctxDone { return tctx.Err() @@ -903,7 +914,8 @@ func (d *Dumper) concurrentDumpTiDBPartitionTables(tctx *tcontext.Context, conn func (d *Dumper) sendConcurrentDumpTiDBTasks(tctx *tcontext.Context, meta TableMeta, taskChan chan<- Task, - handleColNames []string, handleVals [][]string, partition string, startChunkIdx, totalChunk int) error { + handleColNames []string, handleVals [][]string, partition string, + startChunkIdx, totalChunk int) error { db, tbl := meta.DatabaseName(), meta.TableName() if len(handleVals) == 0 { if partition == "" { @@ -919,7 +931,7 @@ func (d *Dumper) sendConcurrentDumpTiDBTasks(tctx *tcontext.Context, for i, w := range where { query := buildSelectQuery(db, tbl, selectField, partition, buildWhereCondition(conf, w), orderByClause) - task := NewTaskTableData(meta, newTableData(query, selectLen, false), i+startChunkIdx, totalChunk) + task := d.newTaskTableData(meta, newTableData(query, selectLen, false), i+startChunkIdx, totalChunk) ctxDone := d.sendTaskToChan(tctx, task, taskChan) if ctxDone { return tctx.Err() @@ -1227,7 +1239,7 @@ func (d *Dumper) dumpSQL(tctx *tcontext.Context, metaConn *BaseConn, taskChan ch conf := d.conf meta := &tableMeta{} data := newTableData(conf.SQL, 0, true) - task := NewTaskTableData(meta, data, 0, 1) + task := d.newTaskTableData(meta, data, 0, 1) c := detectEstimateRows(tctx, metaConn, fmt.Sprintf("EXPLAIN %s", conf.SQL), []string{"rows", "estRows", "count"}) AddCounter(d.metrics.estimateTotalRowsCounter, float64(c)) atomic.StoreInt64(&d.totalTables, int64(1)) @@ -1637,3 +1649,8 @@ func (d *Dumper) renewSelectTableRegionFuncForLowerTiDB(tctx *tcontext.Context) return nil } + +func (d *Dumper) newTaskTableData(meta TableMeta, data TableDataIR, currentChunk, totalChunks int) *TaskTableData { + d.metrics.totalChunks.Add(1) + return NewTaskTableData(meta, data, currentChunk, totalChunks) +} diff --git a/dumpling/export/dump_test.go b/dumpling/export/dump_test.go index 1f18e9e8aa19c..c9a40bba28d6f 100644 --- a/dumpling/export/dump_test.go +++ b/dumpling/export/dump_test.go @@ -18,7 +18,7 @@ import ( "golang.org/x/sync/errgroup" ) -func TestDumpBlock(t *testing.T) { +func TestDumpExit(t *testing.T) { db, mock, err := sqlmock.New() require.NoError(t, err) defer func() { @@ -56,7 +56,6 @@ func TestDumpBlock(t *testing.T) { }) writerCtx := tctx.WithContext(writingCtx) - // simulate taskChan is full taskChan := make(chan Task, 1) taskChan <- &TaskDatabaseMeta{} d.conf.Tables = DatabaseTables{}.AppendTable(database, nil) diff --git a/dumpling/export/metrics.go b/dumpling/export/metrics.go index 5546a614049c4..b4f9aa66ac4d1 100644 --- a/dumpling/export/metrics.go +++ b/dumpling/export/metrics.go @@ -8,6 +8,7 @@ import ( "github.com/pingcap/tidb/util/promutil" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" + "go.uber.org/atomic" ) type metrics struct { @@ -19,6 +20,10 @@ type metrics struct { receiveWriteChunkTimeHistogram *prometheus.HistogramVec errorCount *prometheus.CounterVec taskChannelCapacity *prometheus.GaugeVec + // todo: add these to metrics + totalChunks atomic.Int64 + completedChunks atomic.Int64 + progressReady atomic.Bool } func newMetrics(f promutil.Factory, constLabels prometheus.Labels) *metrics { diff --git a/dumpling/export/status.go b/dumpling/export/status.go index d3f712f5bf491..ec845b051dac7 100644 --- a/dumpling/export/status.go +++ b/dumpling/export/status.go @@ -9,6 +9,7 @@ import ( "time" "github.com/docker/go-units" + "github.com/pingcap/failpoint" tcontext "github.com/pingcap/tidb/dumpling/context" "go.uber.org/zap" ) @@ -17,6 +18,11 @@ const logProgressTick = 2 * time.Minute func (d *Dumper) runLogProgress(tctx *tcontext.Context) { logProgressTicker := time.NewTicker(logProgressTick) + failpoint.Inject("EnableLogProgress", func() { + logProgressTicker.Stop() + logProgressTicker = time.NewTicker(time.Duration(1) * time.Second) + tctx.L().Debug("EnableLogProgress") + }) lastCheckpoint := time.Now() lastBytes := float64(0) defer logProgressTicker.Stop() @@ -34,6 +40,8 @@ func (d *Dumper) runLogProgress(tctx *tcontext.Context) { zap.String("estimate total rows", fmt.Sprintf("%.0f", s.EstimateTotalRows)), zap.String("finished size", units.HumanSize(s.FinishedBytes)), zap.Float64("average speed(MiB/s)", (s.FinishedBytes-lastBytes)/(1048576e-9*nanoseconds)), + zap.Float64("recent speed bps", s.CurrentSpeedBPS), + zap.String("chunks progress", s.Progress), ) lastCheckpoint = time.Now() @@ -49,7 +57,8 @@ type DumpStatus struct { FinishedRows float64 EstimateTotalRows float64 TotalTables int64 - CurrentSpeedBPS int64 + CurrentSpeedBPS float64 + Progress string } // GetStatus returns the status of dumping by reading metrics. @@ -60,7 +69,16 @@ func (d *Dumper) GetStatus() *DumpStatus { ret.FinishedBytes = ReadGauge(d.metrics.finishedSizeGauge) ret.FinishedRows = ReadGauge(d.metrics.finishedRowsGauge) ret.EstimateTotalRows = ReadCounter(d.metrics.estimateTotalRowsCounter) - ret.CurrentSpeedBPS = d.speedRecorder.GetSpeed(int64(ret.FinishedBytes)) + ret.CurrentSpeedBPS = d.speedRecorder.GetSpeed(ret.FinishedBytes) + if d.metrics.progressReady.Load() { + progress := float64(d.metrics.completedChunks.Load()) / float64(d.metrics.totalChunks.Load()) + if progress > 1 { + ret.Progress = "100 %" + d.L().Warn("completedChunks is greater than totalChunks", zap.Int64("completedChunks", d.metrics.completedChunks.Load()), zap.Int64("totalChunks", d.metrics.totalChunks.Load())) + } else { + ret.Progress = fmt.Sprintf("%5.2f %%", progress*100) + } + } return ret } @@ -79,9 +97,9 @@ func calculateTableCount(m DatabaseTables) int { // SpeedRecorder record the finished bytes and calculate its speed. type SpeedRecorder struct { mu sync.Mutex - lastFinished int64 + lastFinished float64 lastUpdateTime time.Time - speedBPS int64 + speedBPS float64 } // NewSpeedRecorder new a SpeedRecorder. @@ -92,7 +110,7 @@ func NewSpeedRecorder() *SpeedRecorder { } // GetSpeed calculate status speed. -func (s *SpeedRecorder) GetSpeed(finished int64) int64 { +func (s *SpeedRecorder) GetSpeed(finished float64) float64 { s.mu.Lock() defer s.mu.Unlock() @@ -103,9 +121,10 @@ func (s *SpeedRecorder) GetSpeed(finished int64) int64 { } now := time.Now() - elapsed := int64(now.Sub(s.lastUpdateTime).Seconds()) + elapsed := now.Sub(s.lastUpdateTime).Seconds() if elapsed == 0 { - elapsed = 1 + // if time is short, return last speed + return s.speedBPS } currentSpeed := (finished - s.lastFinished) / elapsed if currentSpeed == 0 { diff --git a/dumpling/export/status_test.go b/dumpling/export/status_test.go index c6f1f395fb0da..14c08c7c36ba7 100644 --- a/dumpling/export/status_test.go +++ b/dumpling/export/status_test.go @@ -3,6 +3,7 @@ package export import ( + "math" "testing" "time" @@ -35,8 +36,8 @@ func TestGetParameters(t *testing.T) { func TestSpeedRecorder(t *testing.T) { testCases := []struct { spentTime int64 - finished int64 - expected int64 + finished float64 + expected float64 }{ {spentTime: 1, finished: 100, expected: 100}, {spentTime: 2, finished: 200, expected: 50}, @@ -47,6 +48,8 @@ func TestSpeedRecorder(t *testing.T) { for _, tc := range testCases { time.Sleep(time.Duration(tc.spentTime) * time.Second) recentSpeed := speedRecorder.GetSpeed(tc.finished) - require.Equal(t, tc.expected, recentSpeed) + if math.Abs(tc.expected-recentSpeed) > 1 { + require.FailNow(t, "speed is unexpected", "expected: %5.2f, recent: %5.2f", tc.expected, recentSpeed) + } } } diff --git a/dumpling/export/task.go b/dumpling/export/task.go index 36d88c3e3454c..760f5833df017 100644 --- a/dumpling/export/task.go +++ b/dumpling/export/task.go @@ -2,7 +2,9 @@ package export -import "fmt" +import ( + "fmt" +) // Task is a file dump task for dumpling, it could either be dumping database/table/view/policy metadata, table data type Task interface { diff --git a/dumpling/export/util.go b/dumpling/export/util.go index cad703f9f23f4..6c7443a1bee84 100644 --- a/dumpling/export/util.go +++ b/dumpling/export/util.go @@ -78,3 +78,44 @@ func string2Map(a, b []string) map[string]string { func needRepeatableRead(serverType version.ServerType, consistency string) bool { return consistency != ConsistencyTypeSnapshot || serverType != version.ServerTypeTiDB } + +func infiniteChan[T any]() (chan<- T, <-chan T) { + in, out := make(chan T), make(chan T) + + go func() { + var ( + q []T + e T + ok bool + ) + handleRead := func() bool { + if !ok { + for _, e = range q { + out <- e + } + close(out) + return true + } + q = append(q, e) + return false + } + for { + if len(q) > 0 { + select { + case e, ok = <-in: + if handleRead() { + return + } + case out <- q[0]: + q = q[1:] + } + } else { + e, ok = <-in + if handleRead() { + return + } + } + } + }() + return in, out +} diff --git a/dumpling/export/util_test.go b/dumpling/export/util_test.go index 1686a24902825..35de448432cd0 100644 --- a/dumpling/export/util_test.go +++ b/dumpling/export/util_test.go @@ -29,3 +29,17 @@ func TestRepeatableRead(t *testing.T) { require.True(t, rr == expectRepeatableRead, comment) } } + +func TestInfiniteChan(t *testing.T) { + in, out := infiniteChan[int]() + go func() { + for i := 0; i < 10000; i++ { + in <- i + } + }() + for i := 0; i < 10000; i++ { + j := <-out + require.Equal(t, i, j) + } + close(in) +} diff --git a/dumpling/tests/rows_extreme_int/run.sh b/dumpling/tests/rows_extreme_int/run.sh index 625be84150a2b..01799c0f0a420 100644 --- a/dumpling/tests/rows_extreme_int/run.sh +++ b/dumpling/tests/rows_extreme_int/run.sh @@ -33,8 +33,15 @@ run_sql "analyze table $DB_NAME.$TABLE_NAME;" run_sql "analyze table $DB_NAME.$TABLE_NAME2;" # dumping +# test print status +export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/EnableLogProgress=return()" export DUMPLING_TEST_DATABASE=$DB_NAME -run_dumpling --rows 10 --loglevel debug +run_dumpling --rows 10 --loglevel debug -L ${DUMPLING_OUTPUT_DIR}/dumpling.log + +# make sure that dumpling log contains chunks progress infomation +cnt=$(grep -w "chunks progress.*%" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l|awk '{$1=$1;print}') +echo "chunk progress count is ${cnt}" +[ "$cnt" -ge 1 ] cat "$cur/conf/lightning.toml" # use lightning import data to tidb diff --git a/go.mod b/go.mod index 0b7ff4a451cdc..4d247957f5a47 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,14 @@ module github.com/pingcap/tidb go 1.19 require ( - cloud.google.com/go/storage v1.21.0 + cloud.google.com/go/storage v1.22.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 github.com/BurntSushi/toml v1.2.0 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/Jeffail/gabs/v2 v2.5.1 github.com/Masterminds/semver v1.5.0 - github.com/Shopify/sarama v1.29.0 + github.com/Shopify/sarama v1.36.0 github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 github.com/ashanbrown/makezero v1.1.1 github.com/aws/aws-sdk-go v1.44.48 @@ -20,11 +20,11 @@ require ( github.com/cheggaaa/pb/v3 v3.0.8 github.com/cheynewallace/tabby v1.1.1 github.com/cockroachdb/errors v1.8.1 - github.com/cockroachdb/pebble v0.0.0-20210719141320-8c3bd06debb5 + github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 github.com/coocood/freecache v1.2.1 github.com/coreos/go-semver v0.3.0 github.com/daixiang0/gci v0.6.3 - github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37 + github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 github.com/docker/go-units v0.4.0 @@ -55,12 +55,12 @@ require ( github.com/jingyugao/rowserrcheck v1.1.1 github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df github.com/kisielk/errcheck v1.6.2 - github.com/klauspost/compress v1.15.1 + github.com/klauspost/compress v1.15.9 github.com/kyoh86/exportloopref v0.1.8 github.com/mgechev/revive v1.2.4 github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 github.com/nishanths/predeclared v0.2.2 - github.com/opentracing/basictracer-go v1.0.0 + github.com/opentracing/basictracer-go v1.1.0 github.com/opentracing/opentracing-go v1.2.0 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 @@ -70,7 +70,7 @@ require ( github.com/pingcap/kvproto v0.0.0-20221014081430-26e28e6a281a github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 - github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e + github.com/pingcap/tidb/parser v0.0.0-20221013075951-a5c7c039c6c3 github.com/pingcap/tipb v0.0.0-20221020071514-cd933387bcb5 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.13.0 @@ -89,10 +89,10 @@ require ( github.com/tikv/pd/client v0.0.0-20221010134149-d50e5fe43f14 github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144 github.com/twmb/murmur3 v1.1.3 - github.com/uber/jaeger-client-go v2.22.1+incompatible + github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/vbauerster/mpb/v7 v7.5.3 github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f - github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457 + github.com/xitongsys/parquet-go v1.6.0 github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 go.etcd.io/etcd/api/v3 v3.5.2 go.etcd.io/etcd/client/pkg/v3 v3.5.2 @@ -106,16 +106,16 @@ require ( go.uber.org/multierr v1.8.0 go.uber.org/zap v1.23.0 golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e - golang.org/x/net v0.0.0-20220722155237-a158d28d115b - golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 + golang.org/x/net v0.0.0-20220809184613-07c6da5e1ced + golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 + golang.org/x/term v0.0.0-20220411215600-e5f449aeb171 golang.org/x/text v0.3.7 golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 golang.org/x/tools v0.1.12 - google.golang.org/api v0.74.0 - google.golang.org/grpc v1.45.0 + google.golang.org/api v0.84.0 + google.golang.org/grpc v1.48.0 gopkg.in/yaml.v2 v2.4.0 honnef.co/go/tools v0.3.3 sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 @@ -123,12 +123,12 @@ require ( ) require ( - cloud.google.com/go v0.100.2 // indirect - cloud.google.com/go/compute v1.5.0 // indirect - cloud.google.com/go/iam v0.1.1 // indirect + cloud.google.com/go v0.102.0 // indirect + cloud.google.com/go/compute v1.7.0 // indirect + cloud.google.com/go/iam v0.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 // indirect - github.com/DataDog/zstd v1.4.5 // indirect + github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect @@ -148,7 +148,7 @@ require ( github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/eapache/go-resiliency v1.2.0 // indirect + github.com/eapache/go-resiliency v1.3.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect github.com/eapache/queue v1.1.0 // indirect github.com/fatih/structtag v1.2.0 // indirect @@ -160,28 +160,32 @@ require ( github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-cmp v0.5.8 // indirect - github.com/googleapis/gax-go/v2 v2.2.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa // indirect + github.com/googleapis/gax-go/v2 v2.4.0 // indirect + github.com/googleapis/go-type-adapters v1.0.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/gostaticanalysis/analysisutil v0.7.1 // indirect github.com/gostaticanalysis/comment v1.4.2 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/hashicorp/go-uuid v1.0.2 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/hexops/gotextdiff v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect - github.com/jcmturner/gofork v1.0.0 // indirect - github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect + github.com/jcmturner/gofork v1.7.6 // indirect + github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/jonboulle/clockwork v0.2.2 // indirect + github.com/jonboulle/clockwork v0.3.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid v1.3.1 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/kr/text v0.2.0 // indirect - github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/lufia/plan9stats v0.0.0-20220326011226-f1430873d8db // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect @@ -193,7 +197,8 @@ require ( github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/pierrec/lz4 v2.6.1+incompatible // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 // indirect github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 // indirect github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -201,9 +206,9 @@ require ( github.com/prometheus/procfs v0.8.0 // indirect github.com/prometheus/tsdb v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa // indirect github.com/rivo/uniseg v0.4.2 // indirect - github.com/rogpeppe/go-internal v1.6.1 // indirect + github.com/rogpeppe/go-internal v1.8.0 // indirect github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd // indirect github.com/sirupsen/logrus v1.9.0 // indirect @@ -228,12 +233,12 @@ require ( go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect go.opentelemetry.io/otel/trace v0.20.0 // indirect go.opentelemetry.io/proto/otlp v0.7.0 // indirect - golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect + golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect golang.org/x/exp/typeparams v0.0.0-20220613132600-b0d781184e0d // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect + golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb // indirect + google.golang.org/genproto v0.0.0-20220719170305-83ca9fad585f // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 7f2b44c5f2cd0..7b060ebf2c012 100644 --- a/go.sum +++ b/go.sum @@ -26,9 +26,9 @@ cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= -cloud.google.com/go v0.100.2 h1:t9Iw5QH5v4XtlEQaCtUY7x6sCABps8sW0acw7e2WQ6Y= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0 h1:DAq3r8y4mDgyB/ZPJ9v/5VJNqjgJAxTn6ZYLlUywOu8= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -36,15 +36,17 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= -cloud.google.com/go/compute v1.2.0/go.mod h1:xlogom/6gr8RJGBe7nT2eGsQYAFUbbv8dbC29qE3Xmw= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0 h1:b1zWmYuuHz7gO9kDcM/EpHGr06UgsYNRpNJzI2kFiLM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0 h1:v/k9Eueb8aAJ0vZuxKMrgm6kPhCLZU9HxFU+AFDs9Uk= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/iam v0.1.1 h1:4CapQyNFjiksks1/x7jsvsygFPhihslYk5GptIrlX68= -cloud.google.com/go/iam v0.1.1/go.mod h1:CKqrcnI/suGpybEHxZ7BMehL0oA4LpdyJdUlTl9jVMw= +cloud.google.com/go/iam v0.3.0 h1:exkAomrVUuzx9kWFI1wm3KI0uoDeUFPB4kKGzx6x+Gc= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -54,8 +56,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.21.0 h1:HwnT2u2D309SFDHQII6m18HlrCi3jAXhUMTLOWXYH14= -cloud.google.com/go/storage v1.21.0/go.mod h1:XmRlxkgPjlBONznT2dDUU/5XlpU2OjMnKuqnZI01LAA= +cloud.google.com/go/storage v1.22.1 h1:F6IlQJZrZM++apn9V5/VfS3gbTUYg98PS3EMQAzqtfg= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= @@ -77,8 +79,9 @@ github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EF github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 h1:++HGU87uq9UsSTlFeiOV9uZR3NpYkndUXeYyLv2DTc8= +github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Jeffail/gabs/v2 v2.5.1 h1:ANfZYjpMlfTTKebycu4X1AgkVWumFVDYQl7JwOr4mDk= @@ -91,10 +94,10 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= -github.com/Shopify/sarama v1.29.0 h1:ARid8o8oieau9XrHI55f/L3EoRAhm9px6sonbD7yuUE= -github.com/Shopify/sarama v1.29.0/go.mod h1:2QpgD79wpdAESqNQMxNc0KYMkycd4slxGdV3TWSVqrU= -github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/Shopify/sarama v1.36.0 h1:0OJs3eCcnezkWniVjwBbCJVaa0B1k7ImCRS3WN6NsSk= +github.com/Shopify/sarama v1.36.0/go.mod h1:9glG3eX83tgVYJ5aVtrjVUnEsOPqQIBGx1BWfN+X51I= +github.com/Shopify/toxiproxy/v2 v2.4.0 h1:O1e4Jfvr/hefNTNu+8VtdEG5lSeamJRo4aKhMOKNM64= +github.com/Shopify/toxiproxy/v2 v2.4.0/go.mod h1:3ilnjng821bkozDRxNoo64oI/DKqM+rOyJzb564+bvg= github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= @@ -172,6 +175,7 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292/go.mod h1:qRiX68mZX1lGBkTWyp3CLcenw9I94W2dLeRvMzcn9N4= @@ -186,8 +190,8 @@ github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQ github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/cockroachdb/pebble v0.0.0-20210719141320-8c3bd06debb5 h1:Igd6YmtOZ77EgLAIaE9+mHl7+sAKaZ5m4iMI0Dz/J2A= -github.com/cockroachdb/pebble v0.0.0-20210719141320-8c3bd06debb5/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs= +github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 h1:snjwkhKc/ZtYIC/hg6UoT5PrhXcZmCsaB+z0bonMDcU= +github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3/go.mod h1:buxOO9GBtOcq1DiXDpIPYrmxY020K2A8lOrwno5FetU= github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= @@ -214,6 +218,7 @@ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzA github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -222,8 +227,8 @@ github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1v github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/daixiang0/gci v0.6.3 h1:wUAqXChk8HbwXn8AfxD9DYSCp9Bpz1L3e6Q4Roe+q9E= github.com/daixiang0/gci v0.6.3/go.mod h1:EpVfrztufwVgQRXjnX4zuNinEpLj5OmMjtu/+MB0V0c= -github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37 h1:X6mKGhCFOxrKeeHAjv/3UvT6e5RRxW6wRdlqlV6/H4w= -github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37/go.mod h1:DC3JtzuG7kxMvJ6dZmf2ymjNyoXwgtklr7FN+Um2B0U= +github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 h1:+4P40F8AqFAW4/ft2WXiZXrgtRbS8RLb61D8e6NcMw0= +github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8/go.mod h1:VT5Ecrx/r1oHkQbiEBwkLiuQ51igUBmxXuiw9tnSLqY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -242,8 +247,8 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= -github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= +github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= @@ -261,6 +266,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/etcd-io/gofail v0.0.0-20190801230047-ad7f989257ca/go.mod h1:49H/RkXP8pKaZy4h0d+NW16rSLhyVBt4o6VLJbmOqDE= @@ -286,8 +292,6 @@ github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/form3tech-oss/jwt-go v3.2.6-0.20210809144907-32ab6a8243d7+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= @@ -449,13 +453,19 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa h1:7MYGT2XEMam7Mtzv1yDUYXANedWvwk3HKkR3MyGowy8= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0 h1:s7jOdKSaksJVOxE0Y/S32otcfiP+UQ0cL8/GTKaONwE= github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/go-type-adapters v1.0.0 h1:9XdMn+d/G57qq1s8dNc5IesGCXHf6V2HZ2JwRxfA2tA= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/gophercloud/gophercloud v0.0.0-20190301152420-fca40860790e/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gordonklaus/ineffassign v0.0.0-20210914165742-4cc7213b9bc8 h1:PVRE9d4AQKmbelZ7emNig1+NT27DUmKZn5qXxfio54U= @@ -493,12 +503,15 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack v0.5.4/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= @@ -506,8 +519,9 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv github.com/hashicorp/go-uuid v0.0.0-20180228145832-27454136f036/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= @@ -545,12 +559,12 @@ github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90 github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= -github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= +github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= -github.com/jcmturner/gokrb5/v8 v8.4.2 h1:6ZIM6b/JJN0X8UM43ZOM6Z4SJzla+a/u7scXFJzodkA= -github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= +github.com/jcmturner/gokrb5/v8 v8.4.3 h1:iTonLeSJOn7MVUtyMT+arAn5AKAPrkilzhGw8wE/Tq8= +github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/CyVWsJEsJO0= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jedib0t/go-pretty/v6 v6.2.2 h1:o3McN0rQ4X+IU+HduppSp9TwRdGLRW2rhJXy9CJaCRw= @@ -568,8 +582,9 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df h1:Zrb0IbuLOGHL7nrO2WrcuNWgDTlzFv3zY69QMx4ggQE= github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df/go.mod h1:mAVCUAYtW9NG31eB30umMSLKcDt6mCUWSjoSn5qBh0k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/jonboulle/clockwork v0.3.0 h1:9BSCMi8C+0qdApAp4auwX0RkLGUjs956h0EkuQymUhg= +github.com/jonboulle/clockwork v0.3.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -605,9 +620,8 @@ github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= -github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= @@ -629,8 +643,9 @@ github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvf github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lightstep/lightstep-tracer-go v0.15.6/go.mod h1:6AMpwZpsyCFwSovxzM78e+AsYxE8sGwiM6C3TytaWeI= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/lufia/plan9stats v0.0.0-20220326011226-f1430873d8db h1:m2s7Fwo4OwmcheIWUc/Nw9/MZ0eFtP3to0ovTpqOiCQ= +github.com/lufia/plan9stats v0.0.0-20220326011226-f1430873d8db/go.mod h1:VgrrWVwBO2+6XKn8ypT3WUqvoxCa8R2M5to2tRzGovI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -718,8 +733,9 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/opentracing-contrib/go-stdlib v0.0.0-20170113013457-1de4cc2120e7/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= -github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/basictracer-go v1.1.0 h1:Oa1fTSBvAl8pa3U+IJYqrKm0NALwH9OsgwOqDv4xJW0= +github.com/opentracing/basictracer-go v1.1.0/go.mod h1:V2HZueSJEp879yv285Aap1BS69fQMD+MNP1mRs6mBQc= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= @@ -737,14 +753,14 @@ github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea/go.mod h1:1VcHEd3 github.com/petermattis/goid v0.0.0-20170504144140-0ded85884ba5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= -github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= -github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 h1:MKVFZuqFvAMiDtv3AbihOQ6rY5IE8LWflI1BuZ/hF0Y= github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580/go.mod h1:upwDfet29M5y5koWilbWWA6ca3Lr0YVuzwX/DK58Vdk= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= -github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4 h1:iRtOAQ6FXkY/BGvst3CDfTva4nTqh6CL8WXvanLdbu0= github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= +github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 h1:HVl5539r48eA+uDuX/ziBmQCxzT1pGrzWbKuXT46Bq0= +github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -774,6 +790,7 @@ github.com/pingcap/tipb v0.0.0-20221020071514-cd933387bcb5 h1:Yoo8j5xQGxjlsC3yt0 github.com/pingcap/tipb v0.0.0-20221020071514-cd933387bcb5/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 h1:49lOXmGaUpV9Fz3gd7TFZY106KVlPVa5jcYD1gaQf98= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -822,8 +839,8 @@ github.com/prometheus/tsdb v0.8.0 h1:w1tAGxsBMLkuGrFMhqgcCeBkM5d1YI24udArs+aASuQ github.com/prometheus/tsdb v0.8.0/go.mod h1:fSI0j+IUQrDd7+ZtR9WKIGtoYAYAJUKcKhYLG25tN4g= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa h1:tEkEyxYeZ43TR55QU/hsIt9aRGBxbgGuz9CGykjvogY= +github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8= @@ -832,8 +849,9 @@ github.com/rlmcpherson/s3gof3r v0.5.0/go.mod h1:s7vv7SMDPInkitQMuZzH615G7yWHdrU2 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rubyist/circuitbreaker v2.2.1+incompatible/go.mod h1:Ycs3JgJADPuzJDwffe12k6BZT8hxVi6lFK+gWYJLN4A= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -863,6 +881,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -890,6 +909,7 @@ github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4Cc github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -927,12 +947,13 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/twmb/murmur3 v1.1.3 h1:D83U0XYKcHRYwYIpBKf3Pks91Z0Byda/9SJ8B6EMRcA= github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= -github.com/uber/jaeger-client-go v2.22.1+incompatible h1:NHcubEkVbahf9t3p75TOCR83gdUHXjRJvjoBh1yACsM= -github.com/uber/jaeger-client-go v2.22.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= +github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= @@ -942,16 +963,17 @@ github.com/vbauerster/mpb/v7 v7.5.3 h1:BkGfmb6nMrrBQDFECR/Q7RkKCw7ylMetCb4079CGs github.com/vbauerster/mpb/v7 v7.5.3/go.mod h1:i+h4QY6lmLvBNK2ah1fSreiw3ajskRlBp9AhY/PnuOE= github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f h1:9DDCDwOyEy/gId+IEMrFHLuQ5R/WV0KNxWLler8X2OY= github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f/go.mod h1:8sdOQnirw1PrcnTJYkmW1iOHtUmblMmGdUOHyWYycLI= -github.com/xdg/scram v1.0.3/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww= -github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457 h1:tBbuFCtyJNKT+BFAv6qjvTFpVdy97IYNaBwGUXifIUs= -github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457/go.mod h1:pheqtXeHQFzxJk45lRQ0UIGIivKnLXvialZSFWs81A8= +github.com/xitongsys/parquet-go v1.6.0 h1:j6YrTVZdQx5yywJLIOklZcKVsCoSD1tqOVRXyTBFSjs= +github.com/xitongsys/parquet-go v1.6.0/go.mod h1:pheqtXeHQFzxJk45lRQ0UIGIivKnLXvialZSFWs81A8= github.com/xitongsys/parquet-go-source v0.0.0-20190524061010-2b72cbee77d5/go.mod h1:xxCx7Wpym/3QCo6JhujJX51dzSXrwmb0oH6FQb39SEA= github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 h1:a742S4V5A15F93smuVxA60LQWsrCnN8bKeWDBARU1/k= github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE= @@ -1056,11 +1078,9 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1139,6 +1159,7 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -1157,17 +1178,21 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210427231257-85d9c07bbe3a/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= +golang.org/x/net v0.0.0-20220809184613-07c6da5e1ced h1:3dYNDff0VT5xj+mbj2XucFst9WKk6PdGOrb9n+SbIvw= +golang.org/x/net v0.0.0-20220809184613-07c6da5e1ced/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1186,8 +1211,9 @@ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE= golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb h1:8tDJ3aechhddbdPAxpycgXHJRMLpk/Ab+aa4OgdN5/g= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1199,6 +1225,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1276,6 +1303,7 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210909193231-528a39cd75f3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1287,15 +1315,22 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220411215600-e5f449aeb171 h1:EH1Deb8WZJ0xc0WK//leUHXcX9aLE5SymusoTmMZye8= +golang.org/x/term v0.0.0-20220411215600-e5f449aeb171/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1394,8 +1429,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= @@ -1433,14 +1470,15 @@ google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqiv google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= -google.golang.org/api v0.64.0/go.mod h1:931CdxA8Rm4t6zqTFGSsgwbAEZ2+GMYurbndwSimebM= -google.golang.org/api v0.66.0/go.mod h1:I1dmXYpX7HGwz/ejRxwQp2qj5bFAz93HiCU1C1oYd9M= google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= -google.golang.org/api v0.69.0/go.mod h1:boanBiw+h5c3s+tBPgEzLDRHfFLWV0qXxRHz3ws7C80= google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0 h1:ExR2D+5TYIrMphWgs5JCgwRhEDlPDXXrLwHHMgPHTXE= google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0 h1:NMB9J4cCxs9xEm+1Z9QiO3eFvn7EnQj3Eo3hN6ugVlg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1491,6 +1529,7 @@ google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= @@ -1512,20 +1551,25 @@ google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220111164026-67b88f271998/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220114231437-d2e6a121cae0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220201184016-50beb8ab5c44/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220211171837-173942840c17/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220216160803-4663080d8bc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb h1:0m9wktIpOxGw+SSKmydXWB3Z3GTfcPP6+q75HCQa6HI= google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220719170305-83ca9fad585f h1:P8EiVSxZwC6xH2niv2N66aqwMtYFg+D54gbjpcqKJtM= +google.golang.org/genproto v0.0.0-20220719170305-83ca9fad585f/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1558,8 +1602,12 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w= +google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1574,6 +1622,7 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= From ba6ae458f8b99a660fe92bd79d99af6d31a4f735 Mon Sep 17 00:00:00 2001 From: YangKeao Date: Tue, 1 Nov 2022 05:40:00 -0400 Subject: [PATCH 13/13] *: add variables and support external timestamp read (#38276) close pingcap/tidb#38274 --- DEPS.bzl | 12 +- domain/domain_sysvars.go | 12 ++ executor/fktest/BUILD.bazel | 1 + go.mod | 6 +- go.sum | 14 +- planner/core/planbuilder.go | 10 ++ session/session.go | 2 +- sessionctx/stmtctx/stmtctx.go | 19 +++ sessionctx/variable/session.go | 3 + sessionctx/variable/sysvar.go | 17 +++ sessionctx/variable/tidb_vars.go | 12 ++ sessionctx/variable/variable.go | 2 +- sessionctx/variable/varsutil.go | 18 ++- sessiontxn/staleread/BUILD.bazel | 3 + sessiontxn/staleread/externalts_test.go | 135 +++++++++++++++++++ sessiontxn/staleread/processor.go | 34 +++++ sessiontxn/staleread/processor_test.go | 88 +++++++++++- sessiontxn/staleread/util.go | 17 +++ store/mockstore/unistore/tikv/mock_region.go | 27 ++++ 19 files changed, 405 insertions(+), 27 deletions(-) create mode 100644 sessiontxn/staleread/externalts_test.go diff --git a/DEPS.bzl b/DEPS.bzl index e9d0d04876024..be3c5c4c6e47f 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -2825,8 +2825,8 @@ def go_deps(): name = "com_github_pingcap_kvproto", build_file_proto_mode = "disable_global", importpath = "github.com/pingcap/kvproto", - sum = "h1:McYxPhA8SHqfUtLfQHHN0fQl4dy93IkhlX4Pp2MKIFA=", - version = "v0.0.0-20221014081430-26e28e6a281a", + sum = "h1:FYgKV9znRQmzVrrJDZ0gUfMIvKLAMU1tu1UKJib8bEQ=", + version = "v0.0.0-20221026112947-f8d61344b172", ) go_repository( name = "com_github_pingcap_log", @@ -3429,15 +3429,15 @@ def go_deps(): name = "com_github_tikv_client_go_v2", build_file_proto_mode = "disable_global", importpath = "github.com/tikv/client-go/v2", - sum = "h1:s8eJEGI4p/fxFwMBkoJ+4FAEQNQhHR47TZmVW+EEtOE=", - version = "v2.0.1-0.20221026083454-6c9c7c7c5815", + sum = "h1:NvQHWk0GeXSLEBbmGMPnDMc0to0a3ogzgIRbTKw8MHI=", + version = "v2.0.1-0.20221031063202-30e803b7082c", ) go_repository( name = "com_github_tikv_pd_client", build_file_proto_mode = "disable_global", importpath = "github.com/tikv/pd/client", - sum = "h1:REQOR1XraH1fT9BCoNBPZs1CAe+w7VPLU+d+si7DLYo=", - version = "v0.0.0-20221010134149-d50e5fe43f14", + sum = "h1:ckPpxKcl75mO2N6a4cJXiZH43hvcHPpqc9dh1TmH1nc=", + version = "v0.0.0-20221031025758-80f0d8ca4d07", ) go_repository( name = "com_github_timakin_bodyclose", diff --git a/domain/domain_sysvars.go b/domain/domain_sysvars.go index 19c02a9572934..da2b3aeac3fd3 100644 --- a/domain/domain_sysvars.go +++ b/domain/domain_sysvars.go @@ -15,6 +15,7 @@ package domain import ( + "context" "strconv" "time" @@ -33,6 +34,9 @@ func (do *Domain) initDomainSysVars() { variable.SetStatsCacheCapacity.Store(do.setStatsCacheCapacity) pdClientDynamicOptionFunc := do.setPDClientDynamicOption variable.SetPDClientDynamicOption.Store(&pdClientDynamicOptionFunc) + + variable.SetExternalTimestamp = do.setExternalTimestamp + variable.GetExternalTimestamp = do.getExternalTimestamp } // setStatsCacheCapacity sets statsCache cap @@ -75,3 +79,11 @@ func (do *Domain) updatePDClient(option pd.DynamicOption, val interface{}) error } return pdClient.UpdateOption(option, val) } + +func (do *Domain) setExternalTimestamp(ctx context.Context, ts uint64) error { + return do.store.GetOracle().SetExternalTimestamp(ctx, ts) +} + +func (do *Domain) getExternalTimestamp(ctx context.Context) (uint64, error) { + return do.store.GetOracle().GetExternalTimestamp(ctx) +} diff --git a/executor/fktest/BUILD.bazel b/executor/fktest/BUILD.bazel index 8a5f60122813b..40237466542e5 100644 --- a/executor/fktest/BUILD.bazel +++ b/executor/fktest/BUILD.bazel @@ -11,6 +11,7 @@ go_test( deps = [ "//config", "//executor", + "//kv", "//meta/autoid", "//parser/model", "//planner/core", diff --git a/go.mod b/go.mod index 4d247957f5a47..76ee052056499 100644 --- a/go.mod +++ b/go.mod @@ -67,7 +67,7 @@ require ( github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059 - github.com/pingcap/kvproto v0.0.0-20221014081430-26e28e6a281a + github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 github.com/pingcap/tidb/parser v0.0.0-20221013075951-a5c7c039c6c3 @@ -85,8 +85,8 @@ require ( github.com/stretchr/testify v1.8.0 github.com/tdakkota/asciicheck v0.1.1 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 - github.com/tikv/client-go/v2 v2.0.1-0.20221026083454-6c9c7c7c5815 - github.com/tikv/pd/client v0.0.0-20221010134149-d50e5fe43f14 + github.com/tikv/client-go/v2 v2.0.1-0.20221031063202-30e803b7082c + github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144 github.com/twmb/murmur3 v1.1.3 github.com/uber/jaeger-client-go v2.30.0+incompatible diff --git a/go.sum b/go.sum index 7b060ebf2c012..1b468866f3891 100644 --- a/go.sum +++ b/go.sum @@ -775,13 +775,11 @@ github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059/go.mod h1:fMRU1BA1y+r89 github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/kvproto v0.0.0-20221014081430-26e28e6a281a h1:McYxPhA8SHqfUtLfQHHN0fQl4dy93IkhlX4Pp2MKIFA= -github.com/pingcap/kvproto v0.0.0-20221014081430-26e28e6a281a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 h1:FYgKV9znRQmzVrrJDZ0gUfMIvKLAMU1tu1UKJib8bEQ= +github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= -github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 h1:URLoJ61DmmY++Sa/yyPEQHG2s/ZBeV1FbIswHEMrdoY= github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 h1:HYbcxtnkN3s5tqrZ/z3eJS4j3Db8wMphEm1q10lY/TM= @@ -930,10 +928,10 @@ github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3 h1:f+jULpR github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY= github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJfDRtkanvQPiooDH8HvJ2FBh+iKT/OmiQQ= github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU= -github.com/tikv/client-go/v2 v2.0.1-0.20221026083454-6c9c7c7c5815 h1:s8eJEGI4p/fxFwMBkoJ+4FAEQNQhHR47TZmVW+EEtOE= -github.com/tikv/client-go/v2 v2.0.1-0.20221026083454-6c9c7c7c5815/go.mod h1:9hmGJFrWdehClHg0lv2cYgzvCUEhwLZkH67/PHl75tg= -github.com/tikv/pd/client v0.0.0-20221010134149-d50e5fe43f14 h1:REQOR1XraH1fT9BCoNBPZs1CAe+w7VPLU+d+si7DLYo= -github.com/tikv/pd/client v0.0.0-20221010134149-d50e5fe43f14/go.mod h1:E/7+Fkqzwsrp4duzJ2gLPqFl6awU7QG+5yFRXaQwimM= +github.com/tikv/client-go/v2 v2.0.1-0.20221031063202-30e803b7082c h1:NvQHWk0GeXSLEBbmGMPnDMc0to0a3ogzgIRbTKw8MHI= +github.com/tikv/client-go/v2 v2.0.1-0.20221031063202-30e803b7082c/go.mod h1:X9s4ct/MLk1sFqe5mU79KClKegLFDTa/FCx3hzexGtk= +github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 h1:ckPpxKcl75mO2N6a4cJXiZH43hvcHPpqc9dh1TmH1nc= +github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A= github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144 h1:kl4KhGNsJIbDHS9/4U9yQo1UcPQM0kOMJHn29EoH/Ro= github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 8886d5c4aab4e..b9372010b9e2d 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -3283,6 +3283,16 @@ func (b *PlanBuilder) buildSimple(ctx context.Context, node ast.StmtNode) (Plan, p.StaleTxnStartTS = readTS // consume read ts here b.ctx.GetSessionVars().TxnReadTS.UseTxnReadTS() + } else if b.ctx.GetSessionVars().EnableExternalTSRead && !b.ctx.GetSessionVars().InRestrictedSQL { + // try to get the stale ts from external timestamp + startTS, err := staleread.GetExternalTimestamp(ctx, b.ctx) + if err != nil { + return nil, err + } + if err := sessionctx.ValidateStaleReadTS(ctx, b.ctx, startTS); err != nil { + return nil, err + } + p.StaleTxnStartTS = startTS } } return p, nil diff --git a/session/session.go b/session/session.go index f43e265462c11..6d902f3d7d68c 100644 --- a/session/session.go +++ b/session/session.go @@ -2220,7 +2220,7 @@ func (s *session) onTxnManagerStmtStartOrRetry(ctx context.Context, node ast.Stm func (s *session) validateStatementReadOnlyInStaleness(stmtNode ast.StmtNode) error { vars := s.GetSessionVars() - if !vars.TxnCtx.IsStaleness && vars.TxnReadTS.PeakTxnReadTS() == 0 { + if !vars.TxnCtx.IsStaleness && vars.TxnReadTS.PeakTxnReadTS() == 0 && !vars.EnableExternalTSRead { return nil } errMsg := "only support read-only statement during read-only staleness transactions" diff --git a/sessionctx/stmtctx/stmtctx.go b/sessionctx/stmtctx/stmtctx.go index 3816ac0964de7..9d5fc86387cee 100644 --- a/sessionctx/stmtctx/stmtctx.go +++ b/sessionctx/stmtctx/stmtctx.go @@ -382,6 +382,8 @@ const ( StmtNowTsCacheKey StmtCacheKey = iota // StmtSafeTSCacheKey is a variable for safeTS calculation/cache of one stmt. StmtSafeTSCacheKey + // StmtExternalTSCacheKey is a variable for externalTS calculation/cache of one stmt. + StmtExternalTSCacheKey ) // GetOrStoreStmtCache gets the cached value of the given key if it exists, otherwise stores the value. @@ -397,6 +399,23 @@ func (sc *StatementContext) GetOrStoreStmtCache(key StmtCacheKey, value interfac return sc.stmtCache.data[key] } +// GetOrEvaluateStmtCache gets the cached value of the given key if it exists, otherwise calculate the value. +func (sc *StatementContext) GetOrEvaluateStmtCache(key StmtCacheKey, valueEvaluator func() (interface{}, error)) (interface{}, error) { + sc.stmtCache.mu.Lock() + defer sc.stmtCache.mu.Unlock() + if sc.stmtCache.data == nil { + sc.stmtCache.data = make(map[StmtCacheKey]interface{}) + } + if _, ok := sc.stmtCache.data[key]; !ok { + value, err := valueEvaluator() + if err != nil { + return nil, err + } + sc.stmtCache.data[key] = value + } + return sc.stmtCache.data[key], nil +} + // ResetInStmtCache resets the cache of given key. func (sc *StatementContext) ResetInStmtCache(key StmtCacheKey) { sc.stmtCache.mu.Lock() diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 94e5fcddc5968..7773d44541edb 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1276,6 +1276,9 @@ type SessionVars struct { // AnalyzePartitionMergeConcurrency indicates concurrency for merging partition stats AnalyzePartitionMergeConcurrency int + // EnableExternalTSRead indicates whether to enable read through external ts + EnableExternalTSRead bool + HookContext // OptPrefixIndexSingleScan indicates whether to do some optimizations to avoid double scan for prefix index. diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 0e82a45a3d967..280786fa4daf3 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -1952,6 +1952,23 @@ var defaultSysVars = []*SysVar{ s.OptPrefixIndexSingleScan = TiDBOptOn(val) return nil }}, + {Scope: ScopeGlobal, Name: TiDBExternalTS, Value: strconv.FormatInt(DefTiDBExternalTS, 10), SetGlobal: func(ctx context.Context, s *SessionVars, val string) error { + ts, err := parseTSFromNumberOrTime(s, val) + if err != nil { + return err + } + return SetExternalTimestamp(ctx, ts) + }, GetGlobal: func(ctx context.Context, s *SessionVars) (string, error) { + ts, err := GetExternalTimestamp(ctx) + if err != nil { + return "", err + } + return strconv.Itoa(int(ts)), err + }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableExternalTSRead, Value: BoolToOnOff(false), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + s.EnableExternalTSRead = TiDBOptOn(val) + return nil + }}, } // FeedbackProbability points to the FeedbackProbability in statistics package. diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 76c205d68647a..96459a9049bf9 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -15,6 +15,7 @@ package variable import ( + "context" "math" "github.com/pingcap/tidb/config" @@ -759,6 +760,9 @@ const ( // TiDBOptPrefixIndexSingleScan indicates whether to do some optimizations to avoid double scan for prefix index. // When set to true, `col is (not) null`(`col` is index prefix column) is regarded as index filter rather than table filter. TiDBOptPrefixIndexSingleScan = "tidb_opt_prefix_index_single_scan" + + // TiDBEnableExternalTSRead indicates whether to enable read through an external ts + TiDBEnableExternalTSRead = "tidb_enable_external_ts_read" ) // TiDB vars that have only global scope @@ -837,6 +841,8 @@ const ( TiDBEnableGOGCTuner = "tidb_enable_gogc_tuner" // TiDBGOGCTunerThreshold is to control the threshold of GOGC tuner. TiDBGOGCTunerThreshold = "tidb_gogc_tuner_threshold" + // TiDBExternalTS is the ts to read through when the `TiDBEnableExternalTsRead` is on + TiDBExternalTS = "tidb_external_ts" ) // TiDB intentional limits @@ -1072,6 +1078,8 @@ const ( // DefTiDBGOGCTunerThreshold is to limit TiDBGOGCTunerThreshold. DefTiDBGOGCTunerThreshold float64 = 0.6 DefTiDBOptPrefixIndexSingleScan = true + DefTiDBExternalTS = 0 + DefTiDBEnableExternalTSRead = false ) // Process global variables. @@ -1149,4 +1157,8 @@ var ( EnableDDL func() error = nil // DisableDDL is the func registered by ddl to disable running ddl in this instance. DisableDDL func() error = nil + // SetExternalTimestamp is the func registered by staleread to set externaltimestamp in pd + SetExternalTimestamp func(ctx context.Context, ts uint64) error + // GetExternalTimestamp is the func registered by staleread to get externaltimestamp from pd + GetExternalTimestamp func(ctx context.Context) (uint64, error) ) diff --git a/sessionctx/variable/variable.go b/sessionctx/variable/variable.go index fbb3aa79126f1..3acecb6b83483 100644 --- a/sessionctx/variable/variable.go +++ b/sessionctx/variable/variable.go @@ -548,7 +548,7 @@ func (sv *SysVar) SkipInit() bool { func (sv *SysVar) SkipSysvarCache() bool { switch sv.Name { case TiDBGCEnable, TiDBGCRunInterval, TiDBGCLifetime, - TiDBGCConcurrency, TiDBGCScanLockMode: + TiDBGCConcurrency, TiDBGCScanLockMode, TiDBExternalTS: return true } return false diff --git a/sessionctx/variable/varsutil.go b/sessionctx/variable/varsutil.go index 811cf65bb50b6..8aaf9e2761054 100644 --- a/sessionctx/variable/varsutil.go +++ b/sessionctx/variable/varsutil.go @@ -392,21 +392,25 @@ func setSnapshotTS(s *SessionVars, sVal string) error { return fmt.Errorf("tidb_read_staleness should be clear before setting tidb_snapshot") } + tso, err := parseTSFromNumberOrTime(s, sVal) + s.SnapshotTS = tso + // tx_read_ts should be mutual exclusive with tidb_snapshot + s.TxnReadTS = NewTxnReadTS(0) + return err +} + +func parseTSFromNumberOrTime(s *SessionVars, sVal string) (uint64, error) { if tso, err := strconv.ParseUint(sVal, 10, 64); err == nil { - s.SnapshotTS = tso - return nil + return tso, nil } t, err := types.ParseTime(s.StmtCtx, sVal, mysql.TypeTimestamp, types.MaxFsp) if err != nil { - return err + return 0, err } t1, err := t.GoTime(s.Location()) - s.SnapshotTS = oracle.GoTimeToTS(t1) - // tx_read_ts should be mutual exclusive with tidb_snapshot - s.TxnReadTS = NewTxnReadTS(0) - return err + return oracle.GoTimeToTS(t1), err } func setTxnReadTS(s *SessionVars, sVal string) error { diff --git a/sessiontxn/staleread/BUILD.bazel b/sessiontxn/staleread/BUILD.bazel index f89148671792f..9c1e11823e32a 100644 --- a/sessiontxn/staleread/BUILD.bazel +++ b/sessiontxn/staleread/BUILD.bazel @@ -21,6 +21,7 @@ go_library( "//parser/ast", "//parser/mysql", "//sessionctx", + "//sessionctx/stmtctx", "//sessionctx/variable", "//sessiontxn", "//sessiontxn/internal", @@ -36,6 +37,7 @@ go_test( name = "staleread_test", timeout = "short", srcs = [ + "externalts_test.go", "main_test.go", "processor_test.go", "provider_test.go", @@ -48,6 +50,7 @@ go_test( "//kv", "//parser", "//parser/ast", + "//parser/auth", "//sessionctx", "//sessiontxn", "//table/temptable", diff --git a/sessiontxn/staleread/externalts_test.go b/sessiontxn/staleread/externalts_test.go new file mode 100644 index 0000000000000..289c24d820d8f --- /dev/null +++ b/sessiontxn/staleread/externalts_test.go @@ -0,0 +1,135 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package staleread_test + +import ( + "testing" + + "github.com/pingcap/tidb/parser/auth" + "github.com/pingcap/tidb/testkit" + "github.com/stretchr/testify/require" +) + +func TestReadWriteExternalTimestamp(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + + tk.MustQuery("select @@tidb_external_ts").Check(testkit.Rows("0")) + tk.MustExec("set global tidb_external_ts=19980613") + tk.MustQuery("select @@tidb_external_ts").Check(testkit.Rows("19980613")) + tk.MustExec("set global tidb_external_ts=20220930") + tk.MustQuery("select @@tidb_external_ts").Check(testkit.Rows("20220930")) +} + +func TestExternalTimestampRead(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + + tk.MustExec("use test") + tk.MustExec("create table t (id INT NOT NULL,d double,PRIMARY KEY (id))") + tk.MustExec("insert into t values (0, 100)") + tk.MustExec("insert into t values (1, 100)") + tk.MustExec("insert into t values (2, 100)") + tk.MustExec("insert into t values (3, 100)") + tk.MustQuery("select * from t").Check(testkit.Rows("0 100", "1 100", "2 100", "3 100")) + + tk.MustQuery("select @@tidb_external_ts").Check(testkit.Rows("0")) + tk.MustExec("start transaction;set global tidb_external_ts=@@tidb_current_ts;commit;") + tk.MustExec("insert into t values (4, 100)") + // as the `tidb_external_ts` is set an old value, the newest row (4, 100) cannot be read + tk.MustExec("set tidb_enable_external_ts_read=ON") + tk.MustQuery("select * from t").Check(testkit.Rows("0 100", "1 100", "2 100", "3 100")) + + tk.MustExec("set tidb_enable_external_ts_read=OFF") + tk.MustQuery("select * from t").Check(testkit.Rows("0 100", "1 100", "2 100", "3 100", "4 100")) +} + +func TestExternalTimestampReadonly(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil)) + + tk.MustExec("use test") + tk.MustExec("create table t (id INT NOT NULL,PRIMARY KEY (id))") + + tk.MustQuery("select @@tidb_external_ts").Check(testkit.Rows("0")) + tk.MustExec("start transaction;set global tidb_external_ts=@@tidb_current_ts;commit;") + + tk.MustExec("set tidb_enable_external_ts_read=ON") + _, err := tk.Exec("insert into t values (0)") + require.Error(t, err) + + tk.MustExec("set tidb_enable_external_ts_read=OFF") + tk.MustExec("insert into t values (0)") +} + +func TestExternalTimestampReadWithTransaction(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil)) + + tk.MustExec("use test") + tk.MustExec("create table t (id INT NOT NULL,PRIMARY KEY (id))") + + tk.MustQuery("select @@tidb_external_ts").Check(testkit.Rows("0")) + tk.MustExec("start transaction;set global tidb_external_ts=@@tidb_current_ts;commit;") + + tk.MustExec("insert into t values (0)") + tk.MustQuery("select * from t").Check(testkit.Rows("0")) + + tk.MustExec("set tidb_enable_external_ts_read=ON") + tk.MustQuery("select * from t").Check(testkit.Rows()) + tk.MustExec("start transaction") + tk.MustQuery("select * from t").Check(testkit.Rows()) + tk.MustExec("commit") + + tk.MustExec("set tidb_enable_external_ts_read=OFF") + tk.MustExec("start transaction") + tk.MustQuery("select * from t").Check(testkit.Rows("0")) + tk.MustExec("commit") + + tk.MustExec("start transaction") + tk.MustQuery("select * from t").Check(testkit.Rows("0")) + tk.MustExec("set tidb_enable_external_ts_read=ON") + // `tidb_enable_external_ts_read` doesn't affect existing transaction + tk.MustQuery("select * from t").Check(testkit.Rows("0")) + tk.MustExec("set tidb_enable_external_ts_read=OFF") + tk.MustExec("commit") +} + +func TestExternalTimestampNotAffectPrepare(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil)) + + tk.MustExec("use test") + tk.MustExec("create table t (id INT NOT NULL,PRIMARY KEY (id))") + + tk.MustExec("insert into t values (0)") + tk.MustQuery("select * from t").Check(testkit.Rows("0")) + + tk.MustQuery("select @@tidb_external_ts").Check(testkit.Rows("0")) + tk.MustExec("start transaction;set global tidb_external_ts=@@tidb_current_ts;commit;") + + tk.MustExec("insert into t values (1)") + tk.MustQuery("select * from t").Check(testkit.Rows("0", "1")) + + tk.MustExec("set tidb_enable_external_ts_read=on") + tk.MustExec("prepare my_select from 'select * from t'") + tk.MustQuery("execute my_select").Check(testkit.Rows("0")) + tk.MustExec("set tidb_enable_external_ts_read=off") + + tk.MustQuery("execute my_select").Check(testkit.Rows("0", "1")) +} diff --git a/sessiontxn/staleread/processor.go b/sessiontxn/staleread/processor.go index b01a5c69da577..2fc7f3806632b 100644 --- a/sessiontxn/staleread/processor.go +++ b/sessiontxn/staleread/processor.go @@ -105,6 +105,16 @@ func (p *baseProcessor) setEvaluatedTS(ts uint64) (err error) { }) } +// setEvaluatedTSWithoutEvaluator sets the ts, but not set the evaluator, so it doesn't affect prepare statement +func (p *baseProcessor) setEvaluatedTSWithoutEvaluator(ts uint64) (err error) { + is, err := GetSessionSnapshotInfoSchema(p.sctx, ts) + if err != nil { + return err + } + + return p.setEvaluatedValues(ts, is, nil) +} + func (p *baseProcessor) setEvaluatedEvaluator(evaluator StalenessTSEvaluator) error { ts, err := evaluator(p.sctx) if err != nil { @@ -206,6 +216,10 @@ func (p *staleReadProcessor) evaluateFromTxn() error { nil, ) } + + // Don't consider external ts, but just set non-stale read directly,because stepping here means + // when the transaction begins, the external ts read hasn't been turned on, but it was turned + // on during the transaction. Ignore it to avoid unexpected stepping back. return p.setAsNonStaleRead() } @@ -236,6 +250,14 @@ func (p *staleReadProcessor) evaluateFromStmtTSOrSysVariable(stmtTS uint64) erro return p.setEvaluatedEvaluator(evaluator) } + ts, err := getTSFromExternalTS(p.ctx, p.sctx) + if err != nil { + return errAsOf.FastGenWithCause(err.Error()) + } + if ts > 0 { + return p.setEvaluatedTSWithoutEvaluator(ts) + } + // Otherwise, it means we should not use stale read. return p.setAsNonStaleRead() } @@ -268,6 +290,18 @@ func getTsEvaluatorFromReadStaleness(sctx sessionctx.Context) StalenessTSEvaluat } } +func getTSFromExternalTS(ctx context.Context, sctx sessionctx.Context) (uint64, error) { + if sctx.GetSessionVars().EnableExternalTSRead && !sctx.GetSessionVars().InRestrictedSQL { + externalTimestamp, err := GetExternalTimestamp(ctx, sctx) + if err != nil { + return 0, err + } + return externalTimestamp, nil + } + + return 0, nil +} + // GetSessionSnapshotInfoSchema returns the session's information schema with specified ts func GetSessionSnapshotInfoSchema(sctx sessionctx.Context, snapshotTS uint64) (infoschema.InfoSchema, error) { is, err := domain.GetDomain(sctx).GetSnapshotInfoSchema(snapshotTS) diff --git a/sessiontxn/staleread/processor_test.go b/sessiontxn/staleread/processor_test.go index 5eb9f4aa89936..204bb63a3d8de 100644 --- a/sessiontxn/staleread/processor_test.go +++ b/sessiontxn/staleread/processor_test.go @@ -17,6 +17,7 @@ package staleread_test import ( "context" "fmt" + "strconv" "testing" "time" @@ -93,6 +94,14 @@ func astTableWithAsOf(t *testing.T, dt string) *ast.TableName { return sel.From.TableRefs.Left.(*ast.TableSource).Source.(*ast.TableName) } +func getCurrentExternalTimestamp(t *testing.T, tk *testkit.TestKit) uint64 { + externalTimestampStr := tk.MustQuery("select @@tidb_external_ts").Rows()[0][0].(string) + externalTimestamp, err := strconv.ParseUint(externalTimestampStr, 10, 64) + require.NoError(t, err) + + return externalTimestamp +} + func TestStaleReadProcessorWithSelectTable(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -177,6 +186,46 @@ func TestStaleReadProcessorWithSelectTable(t *testing.T) { require.NoError(t, err) p1.checkMatchProcessor(t, processor, true) tk.MustExec("set @@tidb_read_staleness=''") + + // `@@tidb_external_ts` + tk.MustExec("start transaction;set global tidb_external_ts=@@tidb_current_ts;commit") + tk.MustExec("set tidb_enable_external_ts_read=ON") + processor = createProcessor(t, tk.Session()) + err = processor.OnSelectTable(tn) + require.True(t, processor.IsStaleness()) + expectedTS = getCurrentExternalTimestamp(t, tk) + require.Equal(t, expectedTS, processor.GetStalenessReadTS()) + tk.MustExec("set tidb_enable_external_ts_read=OFF") + + // `@@tidb_external_ts` will be ignored when `as of`, `@@tx_read_ts` or `@@tidb_read_staleness` + tk.MustExec("start transaction;set global tidb_external_ts=@@tidb_current_ts;commit") + tk.MustExec("set tidb_enable_external_ts_read=ON") + processor = createProcessor(t, tk.Session()) + err = processor.OnSelectTable(p1.tn) + require.NoError(t, err) + p1.checkMatchProcessor(t, processor, true) + + tk.MustExec(fmt.Sprintf("SET TRANSACTION READ ONLY AS OF TIMESTAMP '%s'", p1.dt)) + processor = createProcessor(t, tk.Session()) + err = processor.OnSelectTable(tn) + require.NoError(t, err) + p1.checkMatchProcessor(t, processor, true) + + tk.MustExec("set @@tidb_read_staleness=-5") + processor = createProcessor(t, tk.Session()) + err = processor.OnSelectTable(tn) + require.True(t, processor.IsStaleness()) + require.Equal(t, int64(0), processor.GetStalenessInfoSchema().SchemaMetaVersion()) + expectedTS, err = staleread.CalculateTsWithReadStaleness(tk.Session(), -5*time.Second) + require.NoError(t, err) + require.Equal(t, expectedTS, processor.GetStalenessReadTS()) + evaluator = processor.GetStalenessTSEvaluatorForPrepare() + evaluatorTS, err = evaluator(tk.Session()) + require.NoError(t, err) + require.Equal(t, expectedTS, evaluatorTS) + tk.MustExec("set @@tidb_read_staleness=''") + + tk.MustExec("set tidb_enable_external_ts_read=OFF") } func TestStaleReadProcessorWithExecutePreparedStmt(t *testing.T) { @@ -256,6 +305,43 @@ func TestStaleReadProcessorWithExecutePreparedStmt(t *testing.T) { require.NoError(t, err) p1.checkMatchProcessor(t, processor, true) tk.MustExec("set @@tidb_read_staleness=''") + + // `@@tidb_external_ts` + tk.MustExec("start transaction;set global tidb_external_ts=@@tidb_current_ts;commit") + tk.MustExec("set tidb_enable_external_ts_read=ON") + processor = createProcessor(t, tk.Session()) + err = processor.OnExecutePreparedStmt(nil) + require.True(t, processor.IsStaleness()) + expectedTS = getCurrentExternalTimestamp(t, tk) + require.Equal(t, expectedTS, processor.GetStalenessReadTS()) + tk.MustExec("set tidb_enable_external_ts_read=OFF") + + // `@@tidb_external_ts` will be ignored when `as of`, `@@tx_read_ts` or `@@tidb_read_staleness` + tk.MustExec("start transaction;set global tidb_external_ts=@@tidb_current_ts;commit") + tk.MustExec("set tidb_enable_external_ts_read=ON") + + processor = createProcessor(t, tk.Session()) + err = processor.OnSelectTable(p1.tn) + require.NoError(t, err) + p1.checkMatchProcessor(t, processor, true) + + tk.MustExec(fmt.Sprintf("SET TRANSACTION READ ONLY AS OF TIMESTAMP '%s'", p1.dt)) + processor = createProcessor(t, tk.Session()) + err = processor.OnExecutePreparedStmt(nil) + require.NoError(t, err) + p1.checkMatchProcessor(t, processor, true) + + tk.MustExec("set @@tidb_read_staleness=-5") + processor = createProcessor(t, tk.Session()) + err = processor.OnExecutePreparedStmt(nil) + require.True(t, processor.IsStaleness()) + require.Equal(t, int64(0), processor.GetStalenessInfoSchema().SchemaMetaVersion()) + expectedTS, err = staleread.CalculateTsWithReadStaleness(tk.Session(), -5*time.Second) + require.NoError(t, err) + require.Equal(t, expectedTS, processor.GetStalenessReadTS()) + tk.MustExec("set @@tidb_read_staleness=''") + + tk.MustExec("set tidb_enable_external_ts_read=OFF") } func TestStaleReadProcessorInTxn(t *testing.T) { @@ -288,7 +374,7 @@ func TestStaleReadProcessorInTxn(t *testing.T) { require.Error(t, err) require.Equal(t, "[planner:8135]invalid as of timestamp: as of timestamp can't be set in transaction.", err.Error()) - // return an error when execute prepared stmt with ts evaluator + // return an error when execute prepared stmt with as of processor = createProcessor(t, tk.Session()) err = processor.OnExecutePreparedStmt(func(sctx sessionctx.Context) (uint64, error) { return p1.ts, nil diff --git a/sessiontxn/staleread/util.go b/sessiontxn/staleread/util.go index 36d0b869e38f7..3fa84f72cae0b 100644 --- a/sessiontxn/staleread/util.go +++ b/sessiontxn/staleread/util.go @@ -15,12 +15,15 @@ package staleread import ( + "context" "time" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/sessionctx/stmtctx" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" "github.com/tikv/client-go/v2/oracle" ) @@ -65,3 +68,17 @@ func CalculateTsWithReadStaleness(sctx sessionctx.Context, readStaleness time.Du func IsStmtStaleness(sctx sessionctx.Context) bool { return sctx.GetSessionVars().StmtCtx.IsStaleness } + +// GetExternalTimestamp returns the external timestamp in cache, or get and store it in cache +func GetExternalTimestamp(ctx context.Context, sctx sessionctx.Context) (uint64, error) { + // Try to get from the stmt cache to make sure this function is deterministic. + stmtCtx := sctx.GetSessionVars().StmtCtx + externalTimestamp, err := stmtCtx.GetOrEvaluateStmtCache(stmtctx.StmtExternalTSCacheKey, func() (interface{}, error) { + return variable.GetExternalTimestamp(ctx) + }) + + if err != nil { + return 0, errAsOf.FastGenWithCause(err.Error()) + } + return externalTimestamp.(uint64), nil +} diff --git a/store/mockstore/unistore/tikv/mock_region.go b/store/mockstore/unistore/tikv/mock_region.go index efbd48bb612b3..2e3e4fa01ddb1 100644 --- a/store/mockstore/unistore/tikv/mock_region.go +++ b/store/mockstore/unistore/tikv/mock_region.go @@ -36,6 +36,7 @@ import ( "github.com/pingcap/tidb/store/mockstore/unistore/tikv/mvcc" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/util/codec" + "github.com/tikv/client-go/v2/oracle" pdclient "github.com/tikv/pd/client" "golang.org/x/exp/slices" ) @@ -670,6 +671,8 @@ func (rm *MockRegionManager) AddPeer(regionID, storeID, peerID uint64) { type MockPD struct { rm *MockRegionManager gcSafePoint uint64 + + externalTimestamp atomic.Uint64 } // NewMockPD returns a new MockPD. @@ -784,6 +787,30 @@ func (pd *MockPD) UpdateGCSafePoint(ctx context.Context, safePoint uint64) (uint // StoreHeartbeat stores the heartbeat. func (pd *MockPD) StoreHeartbeat(ctx context.Context, stats *pdpb.StoreStats) error { return nil } +// GetExternalTimestamp returns external timestamp +func (pd *MockPD) GetExternalTimestamp(ctx context.Context) (uint64, error) { + return pd.externalTimestamp.Load(), nil +} + +// SetExternalTimestamp sets external timestamp +func (pd *MockPD) SetExternalTimestamp(ctx context.Context, newTimestamp uint64) error { + p, l := GetTS() + currentTSO := oracle.ComposeTS(p, l) + if newTimestamp > currentTSO { + return errors.New("external timestamp is greater than global tso") + } + for { + externalTimestamp := pd.externalTimestamp.Load() + if externalTimestamp > newTimestamp { + return errors.New("cannot decrease the external timestamp") + } + + if pd.externalTimestamp.CompareAndSwap(externalTimestamp, newTimestamp) { + return nil + } + } +} + // Use global variables to prevent pdClients from creating duplicate timestamps. var tsMu = struct { sync.Mutex