Remove unused driver extra classpath upload code (apache-spark-on-k8s#54

)
puneetloya · Mar 7, 2019 · 1b6cece · 1b6cece
1 parent 8f75be2
commit 1b6cece
Show file tree

Hide file tree

Showing 4 changed files with 1 addition and 18 deletions.
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -515,8 +515,6 @@ object SparkSubmit extends CommandLineUtils {
         sysProp = "spark.kubernetes.namespace"),
       OptionAssigner(args.kubernetesUploadJars, KUBERNETES, CLUSTER,
         sysProp = "spark.kubernetes.driver.uploads.jars"),
-      OptionAssigner(args.kubernetesUploadDriverExtraClasspath, KUBERNETES, CLUSTER,
-        sysProp = "spark.kubernetes.driver.uploads.driverExtraClasspath"),
 
         // Other options
       OptionAssigner(args.executorCores, STANDALONE | YARN, ALL_DEPLOY_MODES,

diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
@@ -74,7 +74,6 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
   // Kubernetes only
   var kubernetesNamespace: String = null
   var kubernetesUploadJars: String = null
-  var kubernetesUploadDriverExtraClasspath: String = null
 
   // Standalone cluster mode only
   var supervise: Boolean = false
@@ -204,9 +203,6 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
     kubernetesUploadJars = Option(kubernetesUploadJars)
       .orElse(sparkProperties.get("spark.kubernetes.driver.uploads.jars"))
       .orNull
-    kubernetesUploadDriverExtraClasspath = Option(kubernetesUploadDriverExtraClasspath)
-      .orElse(sparkProperties.get("spark.kubernetes.driver.uploads.driverExtraClasspath"))
-      .orNull
 
     // Try to set main class from JAR if no --class argument is given
     if (mainClass == null && !isPython && !isR && primaryResource != null) {
@@ -451,9 +447,6 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
       case KUBERNETES_UPLOAD_JARS =>
         kubernetesUploadJars = value
 
-      case KUBERNETES_UPLOAD_DRIVER_EXTRA_CLASSPATH =>
-        kubernetesUploadDriverExtraClasspath = value
-
       case HELP =>
         printUsageAndExit(0)
 

diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
@@ -80,8 +80,6 @@ class SparkSubmitOptionParser {
   protected final String KUBERNETES_MASTER = "--kubernetes-master";
   protected final String KUBERNETES_NAMESPACE = "--kubernetes-namespace";
   protected final String KUBERNETES_UPLOAD_JARS = "--upload-jars";
-  protected final String KUBERNETES_UPLOAD_DRIVER_EXTRA_CLASSPATH =
-          "--upload-driver-extra-classpath";
 
   /**
    * This is the canonical list of spark-submit options. Each entry in the array contains the
@@ -124,8 +122,7 @@ class SparkSubmitOptionParser {
     { TOTAL_EXECUTOR_CORES },
     { KUBERNETES_MASTER },
     { KUBERNETES_NAMESPACE },
-    { KUBERNETES_UPLOAD_JARS },
-    { KUBERNETES_UPLOAD_DRIVER_EXTRA_CLASSPATH }
+    { KUBERNETES_UPLOAD_JARS }
   };
 
   /**

diff --git a/...re/src/main/scala/org/apache/spark/deploy/rest/kubernetes/KubernetesSparkRestServer.scala b/...re/src/main/scala/org/apache/spark/deploy/rest/kubernetes/KubernetesSparkRestServer.scala
@@ -135,11 +135,6 @@ private[spark] class KubernetesSparkRestServer(
               } else {
                 val tempDir = Utils.createTempDir()
                 val appResourcePath = resolvedAppResource(appResource, tempDir)
-                val driverClasspathDirectory = new File(tempDir, "driver-extra-classpath")
-                if (!driverClasspathDirectory.mkdir) {
-                  throw new IllegalStateException("Failed to create driver extra classpath" +
-                    s" dir at ${driverClasspathDirectory.getAbsolutePath}")
-                }
                 val jarsDirectory = new File(tempDir, "jars")
                 if (!jarsDirectory.mkdir) {
                   throw new IllegalStateException("Failed to create jars dir at" +