From f32834205f75dbd077224239a0322ed7a0147f3c Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 17 Jan 2023 17:52:23 +0800 Subject: [PATCH 1/4] add new item --- util/execdetails/execdetails.go | 12 ++++++++++-- util/execdetails/execdetails_test.go | 18 ++++++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/util/execdetails/execdetails.go b/util/execdetails/execdetails.go index 1867ebc1c945c..bc88857fe439c 100644 --- a/util/execdetails/execdetails.go +++ b/util/execdetails/execdetails.go @@ -409,7 +409,9 @@ func (crs *CopRuntimeStats) RecordOneCopTask(address string, summary *tipb.Execu totalDmfileSkippedRows: summary.GetTiflashScanContext().GetTotalDmfileSkippedRows(), totalDmfileRoughSetIndexLoadTimeMs: summary.GetTiflashScanContext().GetTotalDmfileRoughSetIndexLoadTimeMs(), totalDmfileReadTimeMs: summary.GetTiflashScanContext().GetTotalDmfileReadTimeMs(), - totalCreateSnapshotTimeMs: summary.GetTiflashScanContext().GetTotalCreateSnapshotTimeMs()}}, threads: int32(summary.GetConcurrency()), + totalCreateSnapshotTimeMs: summary.GetTiflashScanContext().GetTotalCreateSnapshotTimeMs(), + totalLocalRegionNum: summary.GetTiflashScanContext().GetTotalLocalRegionNum(), + totalRemoteRegionNum: summary.GetTiflashScanContext().GetTotalRemoteRegionNum()}}, threads: int32(summary.GetConcurrency()), totalTasks: 1, }) } @@ -538,6 +540,8 @@ type TiFlashScanContext struct { totalDmfileRoughSetIndexLoadTimeMs uint64 totalDmfileReadTimeMs uint64 totalCreateSnapshotTimeMs uint64 + totalLocalRegionNum uint64 + totalRemoteRegionNum uint64 } // Clone implements the deep copy of * TiFlashshScanContext @@ -550,10 +554,12 @@ func (context *TiFlashScanContext) Clone() TiFlashScanContext { totalDmfileRoughSetIndexLoadTimeMs: context.totalDmfileRoughSetIndexLoadTimeMs, totalDmfileReadTimeMs: context.totalDmfileReadTimeMs, totalCreateSnapshotTimeMs: context.totalCreateSnapshotTimeMs, + totalLocalRegionNum: context.totalLocalRegionNum, + totalRemoteRegionNum: context.totalRemoteRegionNum, } } func (context *TiFlashScanContext) String() string { - return fmt.Sprintf("tiflash_scan:{dtfile:{total_scanned_packs:%d, total_skipped_packs:%d, total_scanned_rows:%d, total_skipped_rows:%d, total_rs_index_load_time: %dms, total_read_time: %dms}, total_create_snapshot_time: %dms}", context.totalDmfileScannedPacks, context.totalDmfileSkippedPacks, context.totalDmfileScannedRows, context.totalDmfileSkippedRows, context.totalDmfileRoughSetIndexLoadTimeMs, context.totalDmfileReadTimeMs, context.totalCreateSnapshotTimeMs) + return fmt.Sprintf("tiflash_scan:{dtfile:{total_scanned_packs:%d, total_skipped_packs:%d, total_scanned_rows:%d, total_skipped_rows:%d, total_rs_index_load_time: %dms, total_read_time: %dms}, total_create_snapshot_time: %dms, total_local_region_num: %d, total_remote_region_num: %d}", context.totalDmfileScannedPacks, context.totalDmfileSkippedPacks, context.totalDmfileScannedRows, context.totalDmfileSkippedRows, context.totalDmfileRoughSetIndexLoadTimeMs, context.totalDmfileReadTimeMs, context.totalCreateSnapshotTimeMs, context.totalLocalRegionNum, context.totalRemoteRegionNum) } // Merge make sum to merge the information in TiFlashScanContext @@ -565,6 +571,8 @@ func (context *TiFlashScanContext) Merge(other TiFlashScanContext) { context.totalDmfileRoughSetIndexLoadTimeMs += other.totalDmfileRoughSetIndexLoadTimeMs context.totalDmfileReadTimeMs += other.totalDmfileReadTimeMs context.totalCreateSnapshotTimeMs += other.totalCreateSnapshotTimeMs + context.totalLocalRegionNum += other.totalLocalRegionNum + context.totalRemoteRegionNum += other.totalRemoteRegionNum } // Empty check whether TiFlashScanContext is Empty, if scan no pack and skip no pack, we regard it as empty diff --git a/util/execdetails/execdetails_test.go b/util/execdetails/execdetails_test.go index 88e194637e94d..586029e32e2cb 100644 --- a/util/execdetails/execdetails_test.go +++ b/util/execdetails/execdetails_test.go @@ -135,7 +135,7 @@ func mockExecutorExecutionSummary(TimeProcessedNs, NumProducedRows, NumIteration NumIterations: &NumIterations, XXX_unrecognized: nil} } -func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, NumIterations, Concurrency, totalDmfileScannedPacks, totalDmfileScannedRows, totalDmfileSkippedPacks, totalDmfileSkippedRows, totalDmfileRoughSetIndexLoadTimeMs, totalDmfileReadTimeMs, totalCreateSnapshotTimeMs uint64, ExecutorID string) *tipb.ExecutorExecutionSummary { +func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, NumIterations, Concurrency, totalDmfileScannedPacks, totalDmfileScannedRows, totalDmfileSkippedPacks, totalDmfileSkippedRows, totalDmfileRoughSetIndexLoadTimeMs, totalDmfileReadTimeMs, totalCreateSnapshotTimeMs uint64, totalLocalRegionNum uint64, totalRemoteRegionNum uint64, ExecutorID string) *tipb.ExecutorExecutionSummary { tiflashScanContext := tipb.TiFlashScanContext{ TotalDmfileScannedPacks: &totalDmfileScannedPacks, TotalDmfileSkippedPacks: &totalDmfileSkippedPacks, @@ -144,6 +144,8 @@ func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, Nu TotalDmfileRoughSetIndexLoadTimeMs: &totalDmfileRoughSetIndexLoadTimeMs, TotalDmfileReadTimeMs: &totalDmfileReadTimeMs, TotalCreateSnapshotTimeMs: &totalCreateSnapshotTimeMs, + totalLocalRegionNum: &totalLocalRegionNum, + totalRemoteRegionNum: &totalRemoteRegionNum, } return &tipb.ExecutorExecutionSummary{TimeProcessedNs: &TimeProcessedNs, NumProducedRows: &NumProducedRows, NumIterations: &NumIterations, Concurrency: &Concurrency, ExecutorId: &ExecutorID, DetailInfo: &tipb.ExecutorExecutionSummary_TiflashScanContext{TiflashScanContext: &tiflashScanContext}, XXX_unrecognized: nil} @@ -208,10 +210,10 @@ func TestCopRuntimeStatsForTiFlash(t *testing.T) { tableScanID := 1 aggID := 2 tableReaderID := 3 - stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(1, 1, 1, 1, 1, 8192, 0, 0, 15, 200, 40, "tablescan_"+strconv.Itoa(tableScanID))) - stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(2, 2, 2, 1, 0, 0, 0, 0, 0, 2, 0, "tablescan_"+strconv.Itoa(tableScanID))) - stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(3, 3, 3, 1, 2, 12000, 1, 6000, 60, 1000, 20, "aggregation_"+strconv.Itoa(aggID))) - stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(4, 4, 4, 1, 1, 8192, 10, 80000, 40, 2000, 30, "aggregation_"+strconv.Itoa(aggID))) + stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(1, 1, 1, 1, 1, 8192, 0, 0, 15, 200, 40, 10, 4, "tablescan_"+strconv.Itoa(tableScanID))) + stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(2, 2, 2, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, "tablescan_"+strconv.Itoa(tableScanID))) + stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(3, 3, 3, 1, 2, 12000, 1, 6000, 60, 1000, 20, 5, 1, "aggregation_"+strconv.Itoa(aggID))) + stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(4, 4, 4, 1, 1, 8192, 10, 80000, 40, 2000, 30, 1, 1, "aggregation_"+strconv.Itoa(aggID))) scanDetail := &util.ScanDetail{ TotalKeys: 10, ProcessedKeys: 10, @@ -225,15 +227,15 @@ func TestCopRuntimeStatsForTiFlash(t *testing.T) { require.True(t, stats.ExistsCopStats(tableScanID)) cop := stats.GetOrCreateCopStats(tableScanID, "tiflash") - require.Equal(t, "tiflash_task:{proc max:2ns, min:1ns, avg: 1ns, p80:2ns, p95:2ns, iters:3, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_load_time: 15ms, total_read_time: 202ms}, total_create_snapshot_time: 40ms}", cop.String()) + require.Equal(t, "tiflash_task:{proc max:2ns, min:1ns, avg: 1ns, p80:2ns, p95:2ns, iters:3, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_load_time: 15ms, total_read_time: 202ms}, total_create_snapshot_time: 40ms, total_local_region_num: 10, total_remote_region_num: 4}", cop.String()) copStats := cop.stats["8.8.8.8"] require.NotNil(t, copStats) copStats.SetRowNum(10) copStats.Record(time.Second, 10) - require.Equal(t, "time:1s, loops:2, threads:1, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_load_time: 15ms, total_read_time: 200ms}, total_create_snapshot_time: 40ms}", copStats.String()) - expected := "tiflash_task:{proc max:4ns, min:3ns, avg: 3ns, p80:4ns, p95:4ns, iters:7, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:3, total_skipped_packs:11, total_scanned_rows:20192, total_skipped_rows:86000, total_rs_index_load_time: 100ms, total_read_time: 3000ms}, total_create_snapshot_time: 50ms}" + require.Equal(t, "time:1s, loops:2, threads:1, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_load_time: 15ms, total_read_time: 200ms}, total_create_snapshot_time: 40ms, total_local_region_num: 10, total_remote_region_num: 4}", copStats.String()) + expected := "tiflash_task:{proc max:4ns, min:3ns, avg: 3ns, p80:4ns, p95:4ns, iters:7, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:3, total_skipped_packs:11, total_scanned_rows:20192, total_skipped_rows:86000, total_rs_index_load_time: 100ms, total_read_time: 3000ms}, total_create_snapshot_time: 50ms, total_local_region_num: 6, total_remote_region_num: 2}" require.Equal(t, expected, stats.GetOrCreateCopStats(aggID, "tiflash").String()) rootStats := stats.GetRootStats(tableReaderID) From 39e573649d747b01cdf3c32ea3bda51594dc24c6 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 18 Jan 2023 14:52:58 +0800 Subject: [PATCH 2/4] update dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 65440ab9d18cb..b7bd7c6a11a27 100644 --- a/go.mod +++ b/go.mod @@ -74,7 +74,7 @@ require ( github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e - github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 + github.com/pingcap/tipb v0.0.0-20230118064655-f8511db2f072 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.3.0 diff --git a/go.sum b/go.sum index e34b4c5935340..beacd9984a2d5 100644 --- a/go.sum +++ b/go.sum @@ -794,8 +794,8 @@ github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c h1:crhkw6DD+07Bg1wYh github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 h1:HYbcxtnkN3s5tqrZ/z3eJS4j3Db8wMphEm1q10lY/TM= github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4/go.mod h1:sDCsM39cGiv2vwunZkaFA917vVkqDTGSPbbV7z4Oops= -github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 h1:DbmCfCbcavo0JG+gSp0ySvv1ub/c/j3hsnYzyYPzONo= -github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= +github.com/pingcap/tipb v0.0.0-20230118064655-f8511db2f072 h1:CjfEMb3O5jQSlKXUHnqB6KD8osUroM0fWK0j9AJpoeE= +github.com/pingcap/tipb v0.0.0-20230118064655-f8511db2f072/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 h1:49lOXmGaUpV9Fz3gd7TFZY106KVlPVa5jcYD1gaQf98= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From 0093b1cc3c5d22225aaaf2610eeeebbb1ab8e83f Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 18 Jan 2023 15:26:15 +0800 Subject: [PATCH 3/4] update bzl --- DEPS.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DEPS.bzl b/DEPS.bzl index 442749f514a5e..d105b1789cb97 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -2991,8 +2991,8 @@ def go_deps(): name = "com_github_pingcap_tipb", build_file_proto_mode = "disable_global", importpath = "github.com/pingcap/tipb", - sum = "h1:DbmCfCbcavo0JG+gSp0ySvv1ub/c/j3hsnYzyYPzONo=", - version = "v0.0.0-20221123081521-2fb828910813", + sum = "h1:CjfEMb3O5jQSlKXUHnqB6KD8osUroM0fWK0j9AJpoeE=", + version = "v0.0.0-20230118064655-f8511db2f072", ) go_repository( name = "com_github_pkg_browser", From 92f11d7f6599578650fed7fdf358af0c3e176b6a Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 18 Jan 2023 16:43:48 +0800 Subject: [PATCH 4/4] fix bug --- util/execdetails/execdetails_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/execdetails/execdetails_test.go b/util/execdetails/execdetails_test.go index 586029e32e2cb..21dd8ec8a9a50 100644 --- a/util/execdetails/execdetails_test.go +++ b/util/execdetails/execdetails_test.go @@ -144,8 +144,8 @@ func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, Nu TotalDmfileRoughSetIndexLoadTimeMs: &totalDmfileRoughSetIndexLoadTimeMs, TotalDmfileReadTimeMs: &totalDmfileReadTimeMs, TotalCreateSnapshotTimeMs: &totalCreateSnapshotTimeMs, - totalLocalRegionNum: &totalLocalRegionNum, - totalRemoteRegionNum: &totalRemoteRegionNum, + TotalLocalRegionNum: &totalLocalRegionNum, + TotalRemoteRegionNum: &totalRemoteRegionNum, } return &tipb.ExecutorExecutionSummary{TimeProcessedNs: &TimeProcessedNs, NumProducedRows: &NumProducedRows, NumIterations: &NumIterations, Concurrency: &Concurrency, ExecutorId: &ExecutorID, DetailInfo: &tipb.ExecutorExecutionSummary_TiflashScanContext{TiflashScanContext: &tiflashScanContext}, XXX_unrecognized: nil}