From 10cae04108c375a7f5ca7685fea593bd7f49f7a6 Mon Sep 17 00:00:00 2001 From: Wang Shuo Date: Thu, 2 Jan 2020 16:40:22 -0800 Subject: [PATCH] [SPARK-30285][CORE] Fix deadlock between LiveListenerBus#stop and AsyncEventQueue#removeListenerOnError ### What changes were proposed in this pull request? There is a deadlock between `LiveListenerBus#stop` and `AsyncEventQueue#removeListenerOnError`. We can reproduce as follows: 1. Post some events to `LiveListenerBus` 2. Call `LiveListenerBus#stop` and hold the synchronized lock of `bus`(https://github.com/apache/spark/blob/5e92301723464d0876b5a7eec59c15fed0c5b98c/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala#L229), waiting until all the events are processed by listeners, then remove all the queues 3. Event queue would drain out events by posting to its listeners. If a listener is interrupted, it will call `AsyncEventQueue#removeListenerOnError`, inside it will call `bus.removeListener`(https://github.com/apache/spark/blob/7b1b60c7583faca70aeab2659f06d4e491efa5c0/core/src/main/scala/org/apache/spark/scheduler/AsyncEventQueue.scala#L207), trying to acquire synchronized lock of bus, resulting in deadlock This PR removes the `synchronized` from `LiveListenerBus.stop` because underlying data structures themselves are thread-safe. ### Why are the changes needed? To fix deadlock. ### Does this PR introduce any user-facing change? No. ### How was this patch tested? New UT. Closes #26924 from wangshuo128/event-queue-race-condition. Authored-by: Wang Shuo Signed-off-by: Marcelo Vanzin --- .../spark/scheduler/LiveListenerBus.scala | 6 +- .../spark/scheduler/SparkListenerSuite.scala | 70 +++++++++++++++++++ 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala b/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala index bbbddd86cad39..95b0096cade38 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala @@ -226,10 +226,8 @@ private[spark] class LiveListenerBus(conf: SparkConf) { return } - synchronized { - queues.asScala.foreach(_.stop()) - queues.clear() - } + queues.asScala.foreach(_.stop()) + queues.clear() } // For testing only. diff --git a/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala index a0da3ca5b5f3b..d4e8d63b54e5f 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala @@ -529,6 +529,47 @@ class SparkListenerSuite extends SparkFunSuite with LocalSparkContext with Match } } + Seq(true, false).foreach { throwInterruptedException => + val suffix = if (throwInterruptedException) "throw interrupt" else "set Thread interrupted" + test(s"SPARK-30285: Fix deadlock in AsyncEventQueue.removeListenerOnError: $suffix") { + val LISTENER_BUS_STOP_WAITING_TIMEOUT_MILLIS = 10 * 1000L // 10 seconds + val bus = new LiveListenerBus(new SparkConf(false)) + val counter1 = new BasicJobCounter() + val counter2 = new BasicJobCounter() + val interruptingListener = new DelayInterruptingJobCounter(throwInterruptedException, 3) + bus.addToSharedQueue(counter1) + bus.addToSharedQueue(interruptingListener) + bus.addToEventLogQueue(counter2) + assert(bus.activeQueues() === Set(SHARED_QUEUE, EVENT_LOG_QUEUE)) + assert(bus.findListenersByClass[BasicJobCounter]().size === 2) + assert(bus.findListenersByClass[DelayInterruptingJobCounter]().size === 1) + + bus.start(mockSparkContext, mockMetricsSystem) + + (0 until 5).foreach { jobId => + bus.post(SparkListenerJobEnd(jobId, jobCompletionTime, JobSucceeded)) + } + + // Call bus.stop in a separate thread, otherwise we will block here until bus is stopped + val stoppingThread = new Thread(() => { + bus.stop() + }) + stoppingThread.start() + // Notify interrupting listener starts to work + interruptingListener.sleep = false + // Wait for bus to stop + stoppingThread.join(LISTENER_BUS_STOP_WAITING_TIMEOUT_MILLIS) + + // Stopping has been finished + assert(stoppingThread.isAlive === false) + // All queues are removed + assert(bus.activeQueues() === Set.empty) + assert(counter1.count === 5) + assert(counter2.count === 5) + assert(interruptingListener.count === 3) + } + } + test("event queue size can be configued through spark conf") { // configure the shared queue size to be 1, event log queue size to be 2, // and listner bus event queue size to be 5 @@ -627,6 +668,35 @@ class SparkListenerSuite extends SparkFunSuite with LocalSparkContext with Match } } } + + /** + * A simple listener that works as follows: + * 1. sleep and wait when `sleep` is true + * 2. when `sleep` is false, start to work: + * if it is interruptOnJobId, interrupt + * else count SparkListenerJobEnd numbers + */ + private class DelayInterruptingJobCounter( + val throwInterruptedException: Boolean, + val interruptOnJobId: Int) extends SparkListener { + @volatile var sleep = true + var count = 0 + + override def onJobEnd(jobEnd: SparkListenerJobEnd): Unit = { + while (sleep) { + Thread.sleep(10) + } + if (interruptOnJobId == jobEnd.jobId) { + if (throwInterruptedException) { + throw new InterruptedException("got interrupted") + } else { + Thread.currentThread().interrupt() + } + } else { + count += 1 + } + } + } } // These classes can't be declared inside of the SparkListenerSuite class because we don't want