Merge branch 'add_plan_cost_detail_3' of https://github.com/Yisaer/tidb…

… into add_plan_cost_detail_3

.bazelrc

@@ -1,21 +1,27 @@
 startup --host_jvm_args=-Xmx8g
 startup --unlimit_coredumps
 
+run:ci --color=yes
+
+build --announce_rc
 build --java_language_version=17
 build --java_runtime_version=17
 build --tool_java_language_version=17
 build --tool_java_runtime_version=17
-
-run --color=yes
-build:release --workspace_status_command=./build/print-workspace-status.sh --stamp
-build:release --config=ci
 build --incompatible_strict_action_env --incompatible_enable_cc_toolchain_resolution
+build:ci --color=yes
 build:ci --experimental_remote_cache_compression
-test:ci --verbose_failures
-test:ci --test_env=GO_TEST_WRAP_TESTV=1 --test_verbose_timeout_warnings
-test:ci --test_env=TZ=Asia/Shanghai --test_output=errors --experimental_ui_max_stdouterr_bytes=104857600
-
+build:release --workspace_status_command=./build/print-workspace-status.sh --stamp
+build:release --config=ci
+build:race --config=ci
 build:race --@io_bazel_rules_go//go/config:race --test_env=GORACE=halt_on_error=1 --test_sharding_strategy=disabled
+
+test --test_env=TZ=Asia/Shanghai
+test --test_output=errors --test_summary=testcase
+test:ci --color=yes
+test:ci --verbose_failures --test_verbose_timeout_warnings
+test:ci --test_env=GO_TEST_WRAP_TESTV=1
+test:ci --experimental_ui_max_stdouterr_bytes=104857600
 test:race --test_timeout=1200,6000,18000,72000
 
 try-import /data/bazel

BUILD.bazel

@@ -13,17 +13,3 @@ xcode_version(
     name = "xcode_version",
     version = "10.0",
 )
-
-filegroup(
-    name = "package-srcs",
-    srcs = glob(
-        ["**"],
-        exclude = [
-            "bazel-*/**",
-            ".git/**",
-            ".idea/**",
-        ],
-    ),
-    tags = ["automanaged"],
-    visibility = ["//visibility:private"],
-)

DEPS.bzl

@@ -3366,8 +3366,8 @@ def go_deps():
         name = "com_github_tikv_client_go_v2",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/tikv/client-go/v2",
-        sum = "h1:WFR3seA8YtBhDn47YJSW1P1/lwBIXsk0vALnRVuaL/M=",
-        version = "v2.0.1-0.20220815094724-025596b7a20a",
+        sum = "h1:/nr7P8uzJQ7u3wPEBHCokrsVmuDvi/1x/zI/ydk5n8U=",
+        version = "v2.0.1-0.20220818084834-0d0ae0dcfb1f",
     )
     go_repository(
         name = "com_github_tikv_pd_client",

Makefile

@@ -392,7 +392,7 @@ bazel_ci_prepare:
 
 bazel_prepare:
 	bazel run //:gazelle
-	bazel run //:gazelle -- update-repos -from_file=go.mod -build_file_proto_mode=disable
+	bazel run //:gazelle -- update-repos -from_file=go.mod -to_macro DEPS.bzl%go_deps  -build_file_proto_mode=disable
 
 bazel_test: failpoint-enable bazel_ci_prepare
 	bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) \

br/pkg/lightning/backend/local/BUILD.bazel

@@ -42,6 +42,7 @@ go_library(
         "//parser/mysql",
         "//table",
         "//tablecodec",
+        "//types",
         "//util/codec",
         "//util/engine",
         "//util/hack",

br/pkg/lightning/backend/local/local.go

@@ -58,6 +58,7 @@ import (
 	"github.com/pingcap/tidb/parser/mysql"
 	"github.com/pingcap/tidb/table"
 	"github.com/pingcap/tidb/tablecodec"
+	"github.com/pingcap/tidb/types"
 	"github.com/pingcap/tidb/util/codec"
 	"github.com/pingcap/tidb/util/engine"
 	"github.com/pingcap/tidb/util/mathutil"
@@ -1693,6 +1694,10 @@ func (local *local) ResolveDuplicateRows(ctx context.Context, tbl table.Table, t
 				if err == nil {
 					return nil
 				}
+				if types.ErrBadNumber.Equal(err) {
+					logger.Warn("delete duplicate rows encounter error", log.ShortError(err))
+					return common.ErrResolveDuplicateRows.Wrap(err).GenWithStackByArgs(tableName)
+				}
 				if log.IsContextCanceledError(err) {
 					return err
 				}

br/pkg/lightning/common/errors.go

@@ -81,19 +81,20 @@ var (
 	ErrKVIngestFailed        = errors.Normalize("ingest tikv failed", errors.RFCCodeText("Lightning:KV:ErrKVIngestFailed"))
 	ErrKVRaftProposalDropped = errors.Normalize("raft proposal dropped", errors.RFCCodeText("Lightning:KV:ErrKVRaftProposalDropped"))
 
-	ErrUnknownBackend     = errors.Normalize("unknown backend %s", errors.RFCCodeText("Lightning:Restore:ErrUnknownBackend"))
-	ErrCheckLocalFile     = errors.Normalize("cannot find local file for table: %s engineDir: %s", errors.RFCCodeText("Lightning:Restore:ErrCheckLocalFile"))
-	ErrOpenDuplicateDB    = errors.Normalize("open duplicate db error", errors.RFCCodeText("Lightning:Restore:ErrOpenDuplicateDB"))
-	ErrSchemaNotExists    = errors.Normalize("table `%s`.`%s` schema not found", errors.RFCCodeText("Lightning:Restore:ErrSchemaNotExists"))
-	ErrInvalidSchemaStmt  = errors.Normalize("invalid schema statement: '%s'", errors.RFCCodeText("Lightning:Restore:ErrInvalidSchemaStmt"))
-	ErrCreateSchema       = errors.Normalize("create schema failed, table: %s, stmt: %s", errors.RFCCodeText("Lightning:Restore:ErrCreateSchema"))
-	ErrUnknownColumns     = errors.Normalize("unknown columns in header (%s) for table %s", errors.RFCCodeText("Lightning:Restore:ErrUnknownColumns"))
-	ErrChecksumMismatch   = errors.Normalize("checksum mismatched remote vs local => (checksum: %d vs %d) (total_kvs: %d vs %d) (total_bytes:%d vs %d)", errors.RFCCodeText("Lighting:Restore:ErrChecksumMismatch"))
-	ErrRestoreTable       = errors.Normalize("restore table %s failed", errors.RFCCodeText("Lightning:Restore:ErrRestoreTable"))
-	ErrEncodeKV           = errors.Normalize("encode kv error in file %s at offset %d", errors.RFCCodeText("Lightning:Restore:ErrEncodeKV"))
-	ErrAllocTableRowIDs   = errors.Normalize("allocate table row id error", errors.RFCCodeText("Lightning:Restore:ErrAllocTableRowIDs"))
-	ErrInvalidMetaStatus  = errors.Normalize("invalid meta status: '%s'", errors.RFCCodeText("Lightning:Restore:ErrInvalidMetaStatus"))
-	ErrTableIsChecksuming = errors.Normalize("table '%s' is checksuming", errors.RFCCodeText("Lightning:Restore:ErrTableIsChecksuming"))
+	ErrUnknownBackend       = errors.Normalize("unknown backend %s", errors.RFCCodeText("Lightning:Restore:ErrUnknownBackend"))
+	ErrCheckLocalFile       = errors.Normalize("cannot find local file for table: %s engineDir: %s", errors.RFCCodeText("Lightning:Restore:ErrCheckLocalFile"))
+	ErrOpenDuplicateDB      = errors.Normalize("open duplicate db error", errors.RFCCodeText("Lightning:Restore:ErrOpenDuplicateDB"))
+	ErrSchemaNotExists      = errors.Normalize("table `%s`.`%s` schema not found", errors.RFCCodeText("Lightning:Restore:ErrSchemaNotExists"))
+	ErrInvalidSchemaStmt    = errors.Normalize("invalid schema statement: '%s'", errors.RFCCodeText("Lightning:Restore:ErrInvalidSchemaStmt"))
+	ErrCreateSchema         = errors.Normalize("create schema failed, table: %s, stmt: %s", errors.RFCCodeText("Lightning:Restore:ErrCreateSchema"))
+	ErrUnknownColumns       = errors.Normalize("unknown columns in header (%s) for table %s", errors.RFCCodeText("Lightning:Restore:ErrUnknownColumns"))
+	ErrChecksumMismatch     = errors.Normalize("checksum mismatched remote vs local => (checksum: %d vs %d) (total_kvs: %d vs %d) (total_bytes:%d vs %d)", errors.RFCCodeText("Lighting:Restore:ErrChecksumMismatch"))
+	ErrRestoreTable         = errors.Normalize("restore table %s failed", errors.RFCCodeText("Lightning:Restore:ErrRestoreTable"))
+	ErrEncodeKV             = errors.Normalize("encode kv error in file %s at offset %d", errors.RFCCodeText("Lightning:Restore:ErrEncodeKV"))
+	ErrAllocTableRowIDs     = errors.Normalize("allocate table row id error", errors.RFCCodeText("Lightning:Restore:ErrAllocTableRowIDs"))
+	ErrInvalidMetaStatus    = errors.Normalize("invalid meta status: '%s'", errors.RFCCodeText("Lightning:Restore:ErrInvalidMetaStatus"))
+	ErrTableIsChecksuming   = errors.Normalize("table '%s' is checksuming", errors.RFCCodeText("Lightning:Restore:ErrTableIsChecksuming"))
+	ErrResolveDuplicateRows = errors.Normalize("resolve duplicate rows error on table '%s'", errors.RFCCodeText("Lightning:Restore:ErrResolveDuplicateRows"))
 )
 
 type withStack struct {

br/pkg/task/backup.go

@@ -547,7 +547,7 @@ func ParseTSString(ts string, tzCheck bool) (uint64, error) {
 	if tzCheck {
 		tzIdx, _, _, _, _ := types.GetTimezone(ts)
 		if tzIdx < 0 {
-			return 0, errors.Errorf("must set timezone when using datetime format ts")
+			return 0, errors.Errorf("must set timezone when using datetime format ts, e.g. '2018-05-11 01:42:23+0800'")
 		}
 	}
 	t, err := types.ParseTime(sc, ts, mysql.TypeTimestamp, types.MaxFsp)

cmd/explaintest/r/collation_agg_func_disabled.result

@@ -263,13 +263,13 @@ min(d)
 desc format='brief' select min(d collate utf8mb4_bin) from tt;
 id	estRows	task	access object	operator info
 StreamAgg	1.00	root		funcs:min(Column#8)->Column#6
-└─Projection	1.00	root		cast(collation_agg_func.tt.d, json BINARY)->Column#8
+└─Projection	1.00	root		cast(collation_agg_func.tt.d, longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#8
   └─Projection	1.00	root		collation_agg_func.tt.d
     └─TopN	1.00	root		Column#7, offset:0, count:1
-      └─Projection	1.00	root		collation_agg_func.tt.d, cast(collation_agg_func.tt.d, json BINARY)->Column#7
+      └─Projection	1.00	root		collation_agg_func.tt.d, cast(collation_agg_func.tt.d, longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#7
         └─TableReader	1.00	root		data:TopN
-          └─TopN	1.00	cop[tikv]		cast(collation_agg_func.tt.d, json BINARY), offset:0, count:1
-            └─Selection	8000.00	cop[tikv]		not(isnull(cast(cast(collation_agg_func.tt.d, json BINARY), var_string(4294967295))))
+          └─TopN	1.00	cop[tikv]		cast(collation_agg_func.tt.d, longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin), offset:0, count:1
+            └─Selection	8000.00	cop[tikv]		not(isnull(cast(collation_agg_func.tt.d, longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)))
               └─TableFullScan	10000.00	cop[tikv]	table:tt	keep order:false, stats:pseudo
 select min(d collate utf8mb4_bin) from tt;
 min(d collate utf8mb4_bin)
@@ -288,13 +288,13 @@ max(d)
 desc format='brief' select max(d collate utf8mb4_bin) from tt;
 id	estRows	task	access object	operator info
 StreamAgg	1.00	root		funcs:max(Column#8)->Column#6
-└─Projection	1.00	root		cast(collation_agg_func.tt.d, json BINARY)->Column#8
+└─Projection	1.00	root		cast(collation_agg_func.tt.d, longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#8
   └─Projection	1.00	root		collation_agg_func.tt.d
     └─TopN	1.00	root		Column#7:desc, offset:0, count:1
-      └─Projection	1.00	root		collation_agg_func.tt.d, cast(collation_agg_func.tt.d, json BINARY)->Column#7
+      └─Projection	1.00	root		collation_agg_func.tt.d, cast(collation_agg_func.tt.d, longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#7
         └─TableReader	1.00	root		data:TopN
-          └─TopN	1.00	cop[tikv]		cast(collation_agg_func.tt.d, json BINARY):desc, offset:0, count:1
-            └─Selection	8000.00	cop[tikv]		not(isnull(cast(cast(collation_agg_func.tt.d, json BINARY), var_string(4294967295))))
+          └─TopN	1.00	cop[tikv]		cast(collation_agg_func.tt.d, longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin):desc, offset:0, count:1
+            └─Selection	8000.00	cop[tikv]		not(isnull(cast(collation_agg_func.tt.d, longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)))
               └─TableFullScan	10000.00	cop[tikv]	table:tt	keep order:false, stats:pseudo
 select max(d collate utf8mb4_bin) from tt;
 max(d collate utf8mb4_bin)

cmd/explaintest/r/collation_agg_func_enabled.result

@@ -258,9 +258,19 @@ select min(d) from tt;
 min(d)
 {"A": "A"}
 desc format='brief' select min(d collate utf8mb4_bin) from tt;
-Error 1253: COLLATION 'utf8mb4_bin' is not valid for CHARACTER SET 'binary'
+id	estRows	task	access object	operator info
+StreamAgg	1.00	root		funcs:min(Column#8)->Column#6
+└─Projection	1.00	root		cast(collation_agg_func.tt.d, longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#8
+  └─Projection	1.00	root		collation_agg_func.tt.d
+    └─TopN	1.00	root		Column#7, offset:0, count:1
+      └─Projection	1.00	root		collation_agg_func.tt.d, cast(collation_agg_func.tt.d, longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#7
+        └─TableReader	1.00	root		data:TopN
+          └─TopN	1.00	cop[tikv]		cast(collation_agg_func.tt.d, longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin), offset:0, count:1
+            └─Selection	8000.00	cop[tikv]		not(isnull(cast(collation_agg_func.tt.d, longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)))
+              └─TableFullScan	10000.00	cop[tikv]	table:tt	keep order:false, stats:pseudo
 select min(d collate utf8mb4_bin) from tt;
-Error 1253: COLLATION 'utf8mb4_bin' is not valid for CHARACTER SET 'binary'
+min(d collate utf8mb4_bin)
+{"A": "A"}
 desc format='brief' select max(d) from tt;
 id	estRows	task	access object	operator info
 StreamAgg	1.00	root		funcs:max(collation_agg_func.tt.d)->Column#6
@@ -273,8 +283,18 @@ select max(d) from tt;
 max(d)
 {"c": "c"}
 desc format='brief' select max(d collate utf8mb4_bin) from tt;
-Error 1253: COLLATION 'utf8mb4_bin' is not valid for CHARACTER SET 'binary'
+id	estRows	task	access object	operator info
+StreamAgg	1.00	root		funcs:max(Column#8)->Column#6
+└─Projection	1.00	root		cast(collation_agg_func.tt.d, longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#8
+  └─Projection	1.00	root		collation_agg_func.tt.d
+    └─TopN	1.00	root		Column#7:desc, offset:0, count:1
+      └─Projection	1.00	root		collation_agg_func.tt.d, cast(collation_agg_func.tt.d, longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#7
+        └─TableReader	1.00	root		data:TopN
+          └─TopN	1.00	cop[tikv]		cast(collation_agg_func.tt.d, longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin):desc, offset:0, count:1
+            └─Selection	8000.00	cop[tikv]		not(isnull(cast(collation_agg_func.tt.d, longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)))
+              └─TableFullScan	10000.00	cop[tikv]	table:tt	keep order:false, stats:pseudo
 select max(d collate utf8mb4_bin) from tt;
-Error 1253: COLLATION 'utf8mb4_bin' is not valid for CHARACTER SET 'binary'
+max(d collate utf8mb4_bin)
+{"c": "c"}
 drop database collation_agg_func;
 use test

config/config.go

@@ -169,7 +169,6 @@ type Config struct {
 	Path             string `toml:"path" json:"path"`
 	Socket           string `toml:"socket" json:"socket"`
 	Lease            string `toml:"lease" json:"lease"`
-	RunDDL           bool   `toml:"run-ddl" json:"run-ddl"`
 	SplitTable       bool   `toml:"split-table" json:"split-table"`
 	TokenLimit       uint   `toml:"token-limit" json:"token-limit"`
 	TempDir          string `toml:"temp-dir" json:"temp-dir"`
@@ -215,8 +214,6 @@ type Config struct {
 	RepairTableList []string `toml:"repair-table-list" json:"repair-table-list"`
 	// IsolationRead indicates that the TiDB reads data from which isolation level(engine and label).
 	IsolationRead IsolationRead `toml:"isolation-read" json:"isolation-read"`
-	// MaxServerConnections is the maximum permitted number of simultaneous client connections.
-	MaxServerConnections uint32 `toml:"max-server-connections" json:"max-server-connections"`
 	// NewCollationsEnabledOnFirstBootstrap indicates if the new collations are enabled, it effects only when a TiDB cluster bootstrapped on the first time.
 	NewCollationsEnabledOnFirstBootstrap bool `toml:"new_collations_enabled_on_first_bootstrap" json:"new_collations_enabled_on_first_bootstrap"`
 	// Experimental contains parameters for experimental features.
@@ -274,10 +271,12 @@ type Config struct {
 	// OOMUseTmpStorage unused since bootstrap v93
 	OOMUseTmpStorage bool `toml:"oom-use-tmp-storage" json:"oom-use-tmp-storage"`
 
-	// CheckMb4ValueInUTF8, EnableCollectExecutionInfo, Plugin are deprecated.
+	// These items are deprecated because they are turned into instance system variables.
 	CheckMb4ValueInUTF8        AtomicBool `toml:"check-mb4-value-in-utf8" json:"check-mb4-value-in-utf8"`
 	EnableCollectExecutionInfo bool       `toml:"enable-collect-execution-info" json:"enable-collect-execution-info"`
 	Plugin                     Plugin     `toml:"plugin" json:"plugin"`
+	MaxServerConnections       uint32     `toml:"max-server-connections" json:"max-server-connections"`
+	RunDDL                     bool       `toml:"run-ddl" json:"run-ddl"`
 }
 
 // UpdateTempStoragePath is to update the `TempStoragePath` if port/statusPort was changed
@@ -486,11 +485,12 @@ type Instance struct {
 	ForcePriority         string     `toml:"tidb_force_priority" json:"tidb_force_priority"`
 	MemoryUsageAlarmRatio float64    `toml:"tidb_memory_usage_alarm_ratio" json:"tidb_memory_usage_alarm_ratio"`
 	// EnableCollectExecutionInfo enables the TiDB to collect execution info.
-	EnableCollectExecutionInfo bool       `toml:"tidb_enable_collect_execution_info" json:"tidb_enable_collect_execution_info"`
-	PluginDir                  string     `toml:"plugin_dir" json:"plugin_dir"`
-	PluginLoad                 string     `toml:"plugin_load" json:"plugin_load"`
-	MaxConnections             uint32     `toml:"max_connections" json:"max_connections"`
-	TiDBEnableDDL              AtomicBool `toml:"tidb_enable_ddl" json:"tidb_enable_ddl"`
+	EnableCollectExecutionInfo bool   `toml:"tidb_enable_collect_execution_info" json:"tidb_enable_collect_execution_info"`
+	PluginDir                  string `toml:"plugin_dir" json:"plugin_dir"`
+	PluginLoad                 string `toml:"plugin_load" json:"plugin_load"`
+	// MaxConnections is the maximum permitted number of simultaneous client connections.
+	MaxConnections uint32     `toml:"max_connections" json:"max_connections"`
+	TiDBEnableDDL  AtomicBool `toml:"tidb_enable_ddl" json:"tidb_enable_ddl"`
 }
 
 func (l *Log) getDisableTimestamp() bool {
@@ -1036,6 +1036,8 @@ var removedConfig = map[string]struct{}{
 	"performance.feedback-probability":       {}, // This feature is deprecated
 	"performance.query-feedback-limit":       {},
 	"oom-use-tmp-storage":                    {}, // use tidb_enable_tmp_storage_on_oom
+	"max-server-connections":                 {}, // use sysvar max_connections
+	"run-ddl":                                {}, // use sysvar tidb_enable_ddl
 }
 
 // isAllRemovedConfigItems returns true if all the items that couldn't validate

ddl/db_integration_test.go

@@ -3801,8 +3801,15 @@ func TestCreateTempTableInTxn(t *testing.T) {
 	tk := testkit.NewTestKit(t, store)
 	tk.MustExec("use test")
 	tk.MustExec("begin")
-	tk.MustExec("create temporary table t1(id int)")
-	tk.MustQuery("select * from t1")
+	// new created temporary table should be visible
+	tk.MustExec("create temporary table t1(id int primary key, v int)")
+	tk.MustQuery("select * from t1").Check(testkit.Rows())
+	// new inserted data should be visible
+	tk.MustExec("insert into t1 values(123, 456)")
+	tk.MustQuery("select * from t1 where id=123").Check(testkit.Rows("123 456"))
+	// truncate table will clear data but table still visible
+	tk.MustExec("truncate table t1")
+	tk.MustQuery("select * from t1 where id=123").Check(testkit.Rows())
 	tk.MustExec("commit")
 
 	tk1 := testkit.NewTestKit(t, store)
@@ -3812,6 +3819,16 @@ func TestCreateTempTableInTxn(t *testing.T) {
 	tk1.MustExec("create temporary table t1(id int)")
 	tk1.MustExec("insert into tt select * from t1")
 	tk1.MustExec("drop table tt")
+
+	tk2 := testkit.NewTestKit(t, store)
+	tk2.MustExec("use test")
+	tk2.MustExec("create table t2(id int primary key, v int)")
+	tk2.MustExec("insert into t2 values(234, 567)")
+	tk2.MustExec("begin")
+	// create a new temporary table with the same name will override physical table
+	tk2.MustExec("create temporary table t2(id int primary key, v int)")
+	tk2.MustQuery("select * from t2 where id=234").Check(testkit.Rows())
+	tk2.MustExec("commit")
 }
 
 // See https://github.com/pingcap/tidb/issues/29327