Add metric for chunks currently being trasnferred

Kimahriman · Sep 20, 2024 · 8e9ba42 · 8e9ba42
1 parent bb8294c
commit 8e9ba42
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 0 deletions.
diff --git a/.../network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java b/.../network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java
@@ -363,6 +363,8 @@ protected Ratio getRatio() {
       });
       allMetrics.put("registeredExecutorsSize",
                      (Gauge<Integer>) () -> blockManager.getRegisteredExecutorsSize());
+      allMetrics.put("chunksBeingTransferred",
+                     (Gauge<Long>) () -> streamManager.chunksBeingTransferred());
       allMetrics.put("numActiveConnections", activeConnections);
       allMetrics.put("numCaughtExceptions", caughtExceptions);
     }

diff --git a/...-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala b/...-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala
@@ -1082,6 +1082,7 @@ abstract class YarnShuffleServiceSuite extends SparkFunSuite with Matchers {
       "blockTransferMessageRate",
       "blockTransferRateBytes",
       "blockTransferAvgSize_1min",
+      "chunksBeingTransferred",
       "numActiveConnections",
       "numCaughtExceptions",
       "numRegisteredConnections",