Skip to content

Commit

Permalink
Check for user jars/files existence before creating the driver pod. (a…
Browse files Browse the repository at this point in the history
…pache-spark-on-k8s#86)

* Check for user jars/files existence before creating the driver pod.

Close apache-spark-on-k8s#85

* CR
  • Loading branch information
lins05 authored and Puneet Loya committed Mar 7, 2019
1 parent e983e88 commit 3fb535f
Showing 1 changed file with 19 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,10 @@ private[spark] class Client(

def run(): Unit = {
logInfo(s"Starting application $kubernetesAppId in Kubernetes...")
val (driverSubmitSslOptions, isKeyStoreLocalFile) = parseDriverSubmitSslOptions()

Seq(uploadedFiles, uploadedJars, Some(mainAppResource)).foreach(checkForFilesExistence)

val (driverSubmitSslOptions, isKeyStoreLocalFile) = parseDriverSubmitSslOptions()
val parsedCustomLabels = parseCustomLabels(customLabels)
var k8ConfBuilder = new K8SConfigBuilder()
.withApiVersion("v1")
Expand Down Expand Up @@ -661,6 +663,22 @@ private[spark] class Client(
}).toMap
}).getOrElse(Map.empty[String, String])
}

private def checkForFilesExistence(maybePaths: Option[String]): Unit = {
maybePaths.foreach { paths =>
paths.split(",").foreach { path =>
val uri = Utils.resolveURI(path)
uri.getScheme match {
case "file" | null =>
val file = new File(uri.getPath)
if (!file.isFile) {
throw new SparkException(s"""file "${uri}" does not exist!""")
}
case _ =>
}
}
}
}
}

private[spark] object Client extends Logging {
Expand Down

0 comments on commit 3fb535f

Please sign in to comment.