Optimize performance for partitions check

MediaV · Mar 3, 2014 · 3348619 · 3348619
1 parent 61e5a87
commit 3348619
Showing 1 changed file with 4 additions and 4 deletions.
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -847,8 +847,8 @@ class SparkContext(
       partitions: Seq[Int],
       allowLocal: Boolean,
       resultHandler: (Int, U) => Unit) {
-    val rddPartitions = rdd.partitions.map(_.index)
-    require(partitions.forall(rddPartitions.contains(_)), "partition index out of range")
+    val partitionRange = (0 until rdd.partitions.size)
+    require(partitions.forall(partitionRange.contains(_)), "partition index out of range")
     val callSite = getCallSite
     val cleanedFunc = clean(func)
     logInfo("Starting job: " + callSite)
@@ -952,8 +952,8 @@ class SparkContext(
       resultHandler: (Int, U) => Unit,
       resultFunc: => R): SimpleFutureAction[R] =
   {
-    val rddPartitions = rdd.partitions.map(_.index)
-    require(partitions.forall(rddPartitions.contains(_)), "partition index out of range")
+    val partitionRange = (0 until rdd.partitions.size)
+    require(partitions.forall(partitionRange.contains(_)), "partition index out of range")
     val cleanF = clean(processPartition)
     val callSite = getCallSite
     val waiter = dagScheduler.submitJob(