From 8a5eb5068104f527426fb2d0908f45c8eff0749f Mon Sep 17 00:00:00 2001 From: Andrew Ash Date: Fri, 8 Sep 2017 23:33:15 -0700 Subject: [PATCH] [SPARK-21941] Stop storing unused attemptId in SQLTaskMetrics ## What changes were proposed in this pull request? In a driver heap dump containing 390,105 instances of SQLTaskMetrics this would have saved me approximately 3.2MB of memory. Since we're not getting any benefit from storing this unused value, let's eliminate it until a future PR makes use of it. ## How was this patch tested? Existing unit tests Author: Andrew Ash Closes #19153 from ash211/aash/trim-sql-listener. --- .../org/apache/spark/sql/execution/ui/SQLListener.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLListener.scala index b4a91230a0012..8c27af374febd 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLListener.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLListener.scala @@ -255,10 +255,8 @@ class SQLListener(conf: SparkConf) extends SparkListener with Logging { // heartbeat reports } case None => - // TODO Now just set attemptId to 0. Should fix here when we can get the attempt - // id from SparkListenerExecutorMetricsUpdate stageMetrics.taskIdToMetricUpdates(taskId) = new SQLTaskMetrics( - attemptId = 0, finished = finishTask, accumulatorUpdates) + finished = finishTask, accumulatorUpdates) } } case None => @@ -478,10 +476,11 @@ private[ui] class SQLStageMetrics( val stageAttemptId: Long, val taskIdToMetricUpdates: mutable.HashMap[Long, SQLTaskMetrics] = mutable.HashMap.empty) + +// TODO Should add attemptId here when we can get it from SparkListenerExecutorMetricsUpdate /** * Store all accumulatorUpdates for a Spark task. */ private[ui] class SQLTaskMetrics( - val attemptId: Long, // TODO not used yet var finished: Boolean, var accumulatorUpdates: Seq[(Long, Any)])