From 163bda018d054a0d41e983e6c448ce3b3e746d35 Mon Sep 17 00:00:00 2001 From: "Jungtaek Lim (HeartSaVioR)" Date: Thu, 9 Jan 2020 09:50:15 +0900 Subject: [PATCH] Fixed my bad --- .../org/apache/spark/deploy/history/EventFilter.scala | 6 +++--- .../spark/deploy/history/EventLogFileCompactor.scala | 8 +++----- .../spark/deploy/history/EventLogFileCompactorSuite.scala | 6 +++--- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventFilter.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventFilter.scala index fbb44efd7430b..a5f2394960b70 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/EventFilter.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/EventFilter.scala @@ -91,10 +91,10 @@ private[spark] object EventFilter extends Logging { event.foreach { e => val results = filters.flatMap(_.acceptFn().lift.apply(e)) - if (results.isEmpty || !results.contains(false)) { - onAccepted(line, e) - } else { + if (results.nonEmpty && results.forall(_ == false)) { onRejected(line, e) + } else { + onAccepted(line, e) } } } catch { diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileCompactor.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileCompactor.scala index ef055ea288b65..80a0a7067a4e4 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileCompactor.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileCompactor.scala @@ -46,7 +46,7 @@ import org.apache.spark.util.Utils * represents approximate rate of filtered-out events. Score is being calculated via applying * heuristic; task events tend to take most size in event log. */ -private[spark] class EventLogFileCompactor( +class EventLogFileCompactor( sparkConf: SparkConf, hadoopConf: Configuration, fs: FileSystem) extends Logging { @@ -200,11 +200,9 @@ private[spark] class EventLogFileCompactor( * @param compactIndex The index of compact file if the compaction is successful. * Otherwise it will be None. */ -private[spark] case class CompactionResult( - code: CompactionResultCode.Value, - compactIndex: Option[Long]) +case class CompactionResult(code: CompactionResultCode.Value, compactIndex: Option[Long]) -private[spark] object CompactionResultCode extends Enumeration { +object CompactionResultCode extends Enumeration { val SUCCESS, NOT_ENOUGH_FILES, LOW_SCORE_FOR_COMPACTION = Value } diff --git a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileCompactorSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileCompactorSuite.scala index 72da1da9fb600..866e610aab980 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileCompactorSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileCompactorSuite.scala @@ -239,9 +239,9 @@ class EventLogFileCompactorSuite extends SparkFunSuite { // filterApplicationEnd: Some(true) & Some(true) => filter in expectedLines += writeEventToWriter(writer, SparkListenerApplicationEnd(0)) - // filterBlockManagerAdded: Some(true) & Some(false) => filter out - writeEventToWriter(writer, SparkListenerBlockManagerAdded(0, BlockManagerId("1", "host1", 1), - 10)) + // filterBlockManagerAdded: Some(true) & Some(false) => filter in + expectedLines += writeEventToWriter(writer, SparkListenerBlockManagerAdded( + 0, BlockManagerId("1", "host1", 1), 10)) // filterApplicationStart: Some(false) & Some(false) => filter out writeEventToWriter(writer, SparkListenerApplicationStart("app", None, 0, "user", None))