Merge branch 'track/master' into dev/functions

.github/workflows/integration-test-with-real-tikv.yml

@@ -11,8 +11,18 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  pessimistic-test:
+  realtikv-test:
     runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        suite:
+          - brietest
+          - pessimistictest
+          - sessiontest
+          - statisticstest
+          - txntest
+
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
@@ -22,14 +32,13 @@ jobs:
         run: |
           # Disable pipelined pessimistic lock temporarily until tikv#11649 is resolved
           echo -e "[pessimistic-txn]\npipelined = false\n" > tikv.toml
-          echo -e "[raftdb]\nmax-open-files = 10240\n" >> tikv.toml
-          echo -e "[rocksdb]\nmax-open-files = 10240\n" >> tikv.toml
+          echo -e "[raftdb]\nmax-open-files = 20480\n" >> tikv.toml
+          echo -e "[rocksdb]\nmax-open-files = 20480\n" >> tikv.toml
           curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
           $HOME/.tiup/bin/tiup playground nightly --mode tikv-slim --kv 3 --pd 3 --without-monitor --kv.config tikv.toml &
           curl --retry-connrefused --retry 5 --retry-delay 5 http://127.0.0.1:2379
-
       - name: Run Tests
         run: |
           export log_level=error
           make failpoint-enable
-          go test ./tests/pessimistictest -v -with-real-tikv -timeout 20m
+          go test ./tests/realtikvtest/${{ matrix.suite }} -v -with-real-tikv -timeout 30m

.github/workflows/license-checker.yml

@@ -8,6 +8,10 @@ on:
     branches:
       - master
 
+concurrency:
+  group: ${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
 jobs:
   check-license:
     runs-on: ubuntu-latest

Dockerfile

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # Builder image
-FROM golang:1.16-alpine as builder
+FROM golang:1.18.1-alpine as builder
 
 RUN apk add --no-cache \
     wget \

bindinfo/bind_cache.go

@@ -18,10 +18,10 @@ import (
 	"errors"
 	"sync"
 
-	"github.com/cznic/mathutil"
 	"github.com/pingcap/tidb/sessionctx/variable"
 	"github.com/pingcap/tidb/util/hack"
 	"github.com/pingcap/tidb/util/kvcache"
+	"github.com/pingcap/tidb/util/mathutil"
 	"github.com/pingcap/tidb/util/memory"
 )
 

bindinfo/handle.go

@@ -44,6 +44,7 @@ import (
 	tablefilter "github.com/pingcap/tidb/util/table-filter"
 	"github.com/pingcap/tidb/util/timeutil"
 	"go.uber.org/zap"
+	"golang.org/x/exp/maps"
 )
 
 // BindHandle is used to handle all global sql bind operations.
@@ -724,9 +725,7 @@ func (h *BindHandle) removeBindRecord(hash string, meta *BindRecord) {
 
 func copyBindRecordUpdateMap(oldMap map[string]*bindRecordUpdate) map[string]*bindRecordUpdate {
 	newMap := make(map[string]*bindRecordUpdate, len(oldMap))
-	for k, v := range oldMap {
-		newMap[k] = v
-	}
+	maps.Copy(newMap, oldMap)
 	return newMap
 }
 
@@ -961,8 +960,7 @@ func GenerateBindSQL(ctx context.Context, stmtNode ast.StmtNode, planHint string
 			withIdx := strings.Index(bindSQL, "WITH")
 			restoreCtx := format.NewRestoreCtx(format.RestoreStringSingleQuotes|format.RestoreSpacesAroundBinaryOperation|format.RestoreStringWithoutCharset|format.RestoreNameBackQuotes, &withSb)
 			restoreCtx.DefaultDB = defaultDB
-			err := n.With.Restore(restoreCtx)
-			if err != nil {
+			if err := n.With.Restore(restoreCtx); err != nil {
 				logutil.BgLogger().Debug("[sql-bind] restore SQL failed", zap.Error(err))
 				return ""
 			}

br/cmd/br/backup.go

@@ -108,7 +108,7 @@ func newFullBackupCommand() *cobra.Command {
 			return runBackupCommand(command, task.FullBackupCmd)
 		},
 	}
-	task.DefineFilterFlags(command, acceptAllTables)
+	task.DefineFilterFlags(command, acceptAllTables, false)
 	return command
 }
 

br/cmd/br/debug.go

@@ -89,6 +89,9 @@ func newCheckSumCommand() *cobra.Command {
 				if err != nil {
 					return errors.Trace(err)
 				}
+				if schema.Table == nil {
+					continue
+				}
 				tblInfo := &model.TableInfo{}
 				err = json.Unmarshal(schema.Table, tblInfo)
 				if err != nil {
@@ -216,6 +219,10 @@ func newBackupMetaValidateCommand() *cobra.Command {
 			tableIDMap := make(map[int64]int64)
 			// Simulate to create table
 			for _, table := range tables {
+				if table.Info == nil {
+					// empty database.
+					continue
+				}
 				indexIDAllocator := mockid.NewIDAllocator()
 				newTable := new(model.TableInfo)
 				tableID, _ := tableIDAllocator.Alloc()
@@ -229,7 +236,7 @@ func newBackupMetaValidateCommand() *cobra.Command {
 						Name: indexInfo.Name,
 					}
 				}
-				rules := restore.GetRewriteRules(newTable, table.Info, 0)
+				rules := restore.GetRewriteRules(newTable, table.Info, 0, true)
 				rewriteRules.Data = append(rewriteRules.Data, rules.Data...)
 				tableIDMap[table.Info.ID] = int64(tableID)
 			}
@@ -322,7 +329,7 @@ func encodeBackupMetaCommand() *cobra.Command {
 			if err := cfg.ParseFromFlags(cmd.Flags()); err != nil {
 				return errors.Trace(err)
 			}
-			_, s, err := task.GetStorage(ctx, &cfg)
+			_, s, err := task.GetStorage(ctx, cfg.Storage, &cfg)
 			if err != nil {
 				return errors.Trace(err)
 			}

br/cmd/br/main.go

@@ -48,8 +48,9 @@ func main() {
 		NewDebugCommand(),
 		NewBackupCommand(),
 		NewRestoreCommand(),
+		NewStreamCommand(),
 	)
-	// Ouputs cmd.Print to stdout.
+	// Outputs cmd.Print to stdout.
 	rootCmd.SetOut(os.Stdout)
 
 	rootCmd.SetArgs(os.Args[1:])

br/cmd/br/restore.go

@@ -24,6 +24,12 @@ func runRestoreCommand(command *cobra.Command, cmdName string) error {
 		return errors.Trace(err)
 	}
 
+	if task.IsStreamRestore(cmdName) {
+		if err := cfg.ParseStreamRestoreFlags(command.Flags()); err != nil {
+			return errors.Trace(err)
+		}
+	}
+
 	ctx := GetDefaultContext()
 	if cfg.EnableOpenTracing {
 		var store *appdash.MemoryStore
@@ -83,6 +89,7 @@ func NewRestoreCommand() *cobra.Command {
 		newDBRestoreCommand(),
 		newTableRestoreCommand(),
 		newRawRestoreCommand(),
+		newStreamRestoreCommand(),
 	)
 	task.DefineRestoreFlags(command.PersistentFlags())
 
@@ -98,7 +105,7 @@ func newFullRestoreCommand() *cobra.Command {
 			return runRestoreCommand(cmd, task.FullRestoreCmd)
 		},
 	}
-	task.DefineFilterFlags(command, filterOutSysAndMemTables)
+	task.DefineFilterFlags(command, filterOutSysAndMemTables, false)
 	return command
 }
 
@@ -141,3 +148,17 @@ func newRawRestoreCommand() *cobra.Command {
 	task.DefineRawRestoreFlags(command)
 	return command
 }
+
+func newStreamRestoreCommand() *cobra.Command {
+	command := &cobra.Command{
+		Use:   "point",
+		Short: "restore data from log until specify commit timestamp",
+		Args:  cobra.NoArgs,
+		RunE: func(command *cobra.Command, _ []string) error {
+			return runRestoreCommand(command, task.PointRestoreCmd)
+		},
+	}
+	task.DefineFilterFlags(command, filterOutSysAndMemTables, true)
+	task.DefineStreamRestoreFlags(command)
+	return command
+}