diff --git a/conf/spark-defaults.conf.template b/conf/spark-defaults.conf.template index 92adb4c9a575a..94427029b94d7 100644 --- a/conf/spark-defaults.conf.template +++ b/conf/spark-defaults.conf.template @@ -6,4 +6,5 @@ # spark.eventLog.enabled true # spark.eventLog.dir hdfs://namenode:8021/directory # spark.serializer org.apache.spark.serializer.KryoSerializer +# spark.driver.memory 5g # spark.executor.extraJavaOptions -XX:+PrintGCDetail -Dkey=value -Dnumbers="one two three" diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitDriverBootstrapper.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitDriverBootstrapper.scala index 8422406b8f075..5d7cabd6ebd28 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitDriverBootstrapper.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitDriverBootstrapper.scala @@ -58,39 +58,45 @@ private[spark] object SparkSubmitDriverBootstrapper { assume(defaultDriverMemory != null, "OUR_JAVA_MEM must be set") assume(deployMode == "client", "SPARK_SUBMIT_DEPLOY_MODE must be \"client\"!") assume(propertiesFile != null, "SPARK_SUBMIT_PROPERTIES_FILE must be set") - assume(bootstrapDriver != null, "SPARK_SUBMIT_BOOTSTRAP_DRIVER must be set!") + assume(bootstrapDriver != null, "SPARK_SUBMIT_BOOTSTRAP_DRIVER must be set") // Parse the properties file for the equivalent spark.driver.* configs val properties = SparkSubmitArguments.getPropertiesFromFile(new File(propertiesFile)).toMap - val confDriverMemory = properties.get("spark.driver.memory").getOrElse(defaultDriverMemory) - val confLibraryPath = properties.get("spark.driver.extraLibraryPath").getOrElse("") - val confClasspath = properties.get("spark.driver.extraClassPath").getOrElse("") - val confJavaOpts = properties.get("spark.driver.extraJavaOptions").getOrElse("") + val confDriverMemory = properties.get("spark.driver.memory") + val confLibraryPath = properties.get("spark.driver.extraLibraryPath") + val confClasspath = properties.get("spark.driver.extraClassPath") + val confJavaOpts = properties.get("spark.driver.extraJavaOptions") // Favor Spark submit arguments over the equivalent configs in the properties file. // Note that we do not actually use the Spark submit values for library path, classpath, // and Java opts here, because we have already captured them in Bash. - val newDriverMemory = submitDriverMemory.getOrElse(confDriverMemory) + + val newDriverMemory = submitDriverMemory + .orElse(confDriverMemory) + .getOrElse(defaultDriverMemory) + val newLibraryPath = if (submitLibraryPath.isDefined) { // SPARK_SUBMIT_LIBRARY_PATH is already captured in JAVA_OPTS "" } else { - "-Djava.library.path=" + confLibraryPath + confLibraryPath.map("-Djava.library.path=" + _).getOrElse("") } + val newClasspath = if (submitClasspath.isDefined) { // SPARK_SUBMIT_CLASSPATH is already captured in CLASSPATH classpath } else { - classpath + sys.props("path.separator") + confClasspath + classpath + confClasspath.map(sys.props("path.separator") + _).getOrElse("") } + val newJavaOpts = if (submitJavaOpts.isDefined) { // SPARK_SUBMIT_OPTS is already captured in JAVA_OPTS javaOpts } else { - javaOpts + " " + confJavaOpts + javaOpts + confJavaOpts.map(" " + _).getOrElse("") } // Build up command