fix rest end point filter to allow logstart event

apache · May 5, 2017 · 93e7a20 · 93e7a20
1 parent dad87a6
commit 93e7a20
Show file tree

Hide file tree

Showing 10 changed files with 34 additions and 31 deletions.
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -445,7 +445,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
     val newAttempts = try {
       val eventsFilter: ReplayEventsFilter = { eventString =>
         eventString.startsWith(APPL_START_EVENT_PREFIX) ||
-          eventString.startsWith(APPL_END_EVENT_PREFIX)
+          eventString.startsWith(APPL_END_EVENT_PREFIX) ||
+          eventString.startsWith(LOG_START_EVENT_PREFIX)
       }
 
       val logPath = fileStatus.getPath()
@@ -738,6 +739,8 @@ private[history] object FsHistoryProvider {
   private val APPL_START_EVENT_PREFIX = "{\"Event\":\"SparkListenerApplicationStart\""
 
   private val APPL_END_EVENT_PREFIX = "{\"Event\":\"SparkListenerApplicationEnd\""
+
+  private val LOG_START_EVENT_PREFIX = "{\"Event\":\"SparkListenerLogStart\""
 }
 
 /**

diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
@@ -121,7 +121,7 @@ private[spark] class SparkUI private (
         lastUpdated = new Date(startTime),
         sparkUser = getSparkUser,
         completed = false,
-        appSparkVersion = ""
+        appSparkVersion = appSparkVersion
       ))
     ))
   }

diff --git a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
@@ -8,7 +8,7 @@
     "duration" : 10671,
     "sparkUser" : "jose",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "2.1.0-SNAPSHOT",
     "endTimeEpoch" : 1479335620587,
     "startTimeEpoch" : 1479335609916,
     "lastUpdatedEpoch" : 0
@@ -23,7 +23,7 @@
     "duration" : 101795,
     "sparkUser" : "jose",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "2.1.0-SNAPSHOT",
     "endTimeEpoch" : 1479252138874,
     "startTimeEpoch" : 1479252037079,
     "lastUpdatedEpoch" : 0
@@ -38,7 +38,7 @@
     "duration" : 10505,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "endTimeEpoch" : 1430917391398,
     "startTimeEpoch" : 1430917380893,
     "lastUpdatedEpoch" : 0
@@ -54,7 +54,7 @@
     "duration" : 57,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "endTimeEpoch" : 1430917380950,
     "startTimeEpoch" : 1430917380893,
     "lastUpdatedEpoch" : 0
@@ -66,7 +66,7 @@
     "duration" : 10,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "endTimeEpoch" : 1430917380890,
     "startTimeEpoch" : 1430917380880,
     "lastUpdatedEpoch" : 0

diff --git a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
@@ -8,7 +8,7 @@
     "duration" : 10671,
     "sparkUser" : "jose",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "2.1.0-SNAPSHOT",
     "endTimeEpoch" : 1479335620587,
     "startTimeEpoch" : 1479335609916,
     "lastUpdatedEpoch" : 0
@@ -23,7 +23,7 @@
     "duration" : 101795,
     "sparkUser" : "jose",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "2.1.0-SNAPSHOT",
     "endTimeEpoch" : 1479252138874,
     "startTimeEpoch" : 1479252037079,
     "lastUpdatedEpoch" : 0
@@ -38,7 +38,7 @@
     "duration" : 10505,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "endTimeEpoch" : 1430917391398,
     "startTimeEpoch" : 1430917380893,
     "lastUpdatedEpoch" : 0
@@ -54,7 +54,7 @@
     "duration" : 57,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "endTimeEpoch" : 1430917380950,
     "startTimeEpoch" : 1430917380893,
     "lastUpdatedEpoch" : 0
@@ -66,7 +66,7 @@
     "duration" : 10,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "endTimeEpoch" : 1430917380890,
     "startTimeEpoch" : 1430917380880,
     "lastUpdatedEpoch" : 0

diff --git a/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json
@@ -8,7 +8,7 @@
     "duration" : 10671,
     "sparkUser" : "jose",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "2.1.0-SNAPSHOT",
     "endTimeEpoch" : 1479335620587,
     "startTimeEpoch" : 1479335609916,
     "lastUpdatedEpoch" : 0
@@ -23,7 +23,7 @@
     "duration" : 101795,
     "sparkUser" : "jose",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "2.1.0-SNAPSHOT",
     "endTimeEpoch" : 1479252138874,
     "startTimeEpoch" : 1479252037079,
     "lastUpdatedEpoch" : 0
@@ -38,7 +38,7 @@
     "duration" : 10505,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "endTimeEpoch" : 1430917391398,
     "startTimeEpoch" : 1430917380893,
     "lastUpdatedEpoch" : 0

diff --git a/core/src/test/resources/HistoryServerExpectations/maxEndDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/maxEndDate_app_list_json_expectation.json
@@ -9,7 +9,7 @@
     "duration" : 57,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "lastUpdatedEpoch" : 0,
     "startTimeEpoch" : 1430917380893,
     "endTimeEpoch" : 1430917380950
@@ -21,7 +21,7 @@
     "duration" : 10,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "lastUpdatedEpoch" : 0,
     "startTimeEpoch" : 1430917380880,
     "endTimeEpoch" : 1430917380890

diff --git a/...resources/HistoryServerExpectations/minDate_and_maxEndDate_app_list_json_expectation.json b/...resources/HistoryServerExpectations/minDate_and_maxEndDate_app_list_json_expectation.json
@@ -9,7 +9,7 @@
     "duration" : 57,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "lastUpdatedEpoch" : 0,
     "startTimeEpoch" : 1430917380893,
     "endTimeEpoch" : 1430917380950
@@ -21,7 +21,7 @@
     "duration" : 10,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "lastUpdatedEpoch" : 0,
     "startTimeEpoch" : 1430917380880,
     "endTimeEpoch" : 1430917380890

diff --git a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
@@ -8,7 +8,7 @@
     "duration" : 10671,
     "sparkUser" : "jose",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "2.1.0-SNAPSHOT",
     "endTimeEpoch" : 1479335620587,
     "startTimeEpoch" : 1479335609916,
     "lastUpdatedEpoch" : 0
@@ -23,7 +23,7 @@
     "duration" : 101795,
     "sparkUser" : "jose",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "2.1.0-SNAPSHOT",
     "endTimeEpoch" : 1479252138874,
     "startTimeEpoch" : 1479252037079,
     "lastUpdatedEpoch" : 0
@@ -38,7 +38,7 @@
     "duration" : 10505,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "endTimeEpoch" : 1430917391398,
     "startTimeEpoch" : 1430917380893,
     "lastUpdatedEpoch" : 0
@@ -54,7 +54,7 @@
     "duration" : 57,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "endTimeEpoch" : 1430917380950,
     "startTimeEpoch" : 1430917380893,
     "lastUpdatedEpoch" : 0
@@ -66,7 +66,7 @@
     "duration" : 10,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "endTimeEpoch" : 1430917380890,
     "startTimeEpoch" : 1430917380880,
     "lastUpdatedEpoch" : 0

diff --git a/...ources/HistoryServerExpectations/minEndDate_and_maxEndDate_app_list_json_expectation.json b/...ources/HistoryServerExpectations/minEndDate_and_maxEndDate_app_list_json_expectation.json
@@ -9,7 +9,7 @@
     "duration" : 57,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "lastUpdatedEpoch" : 0,
     "startTimeEpoch" : 1430917380893,
     "endTimeEpoch" : 1430917380950
@@ -21,7 +21,7 @@
     "duration" : 10,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "lastUpdatedEpoch" : 0,
     "startTimeEpoch" : 1430917380880,
     "endTimeEpoch" : 1430917380890

diff --git a/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json
@@ -8,7 +8,7 @@
     "duration" : 10671,
     "sparkUser" : "jose",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "2.1.0-SNAPSHOT",
     "startTimeEpoch" : 1479335609916,
     "lastUpdatedEpoch" : 0,
     "endTimeEpoch" : 1479335620587
@@ -23,7 +23,7 @@
     "duration" : 101795,
     "sparkUser" : "jose",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "2.1.0-SNAPSHOT",
     "startTimeEpoch" : 1479252037079,
     "lastUpdatedEpoch" : 0,
     "endTimeEpoch" : 1479252138874
@@ -38,7 +38,7 @@
     "duration" : 10505,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "lastUpdatedEpoch" : 0,
     "startTimeEpoch" : 1430917380893,
     "endTimeEpoch" : 1430917391398
@@ -54,7 +54,7 @@
     "duration" : 57,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "lastUpdatedEpoch" : 0,
     "startTimeEpoch" : 1430917380893,
     "endTimeEpoch" : 1430917380950
@@ -66,7 +66,7 @@
     "duration" : 10,
     "sparkUser" : "irashid",
     "completed" : true,
-    "appSparkVersion" : "",
+    "appSparkVersion" : "1.4.0-SNAPSHOT",
     "lastUpdatedEpoch" : 0,
     "startTimeEpoch" : 1430917380880,
     "endTimeEpoch" : 1430917380890