If SPARK_CONF_DIR environment variable is specified, use spark-env.sh

specified in $SPARK_CONF_DIR when starting daemons.
apache · Sep 3, 2014 · 22273b8 · 22273b8
1 parent f2b5b61
commit 22273b8
Show file tree

Hide file tree

Showing 5 changed files with 48 additions and 7 deletions.
diff --git a/sbin/spark-config.sh b/sbin/spark-config.sh
@@ -33,7 +33,7 @@ this="$config_bin/$script"
 
 export SPARK_PREFIX=`dirname "$this"`/..
 export SPARK_HOME=${SPARK_PREFIX}
-export SPARK_CONF_DIR="$SPARK_HOME/conf"
+export SPARK_CONF_DIR="${SPARK_CONF_DIR:-$SPARK_HOME/conf}"
 # Add the PySpark classes to the PYTHONPATH:
 export PYTHONPATH=$SPARK_HOME/python:$PYTHONPATH
 export PYTHONPATH=$SPARK_HOME/python/lib/py4j-0.8.2.1-src.zip:$PYTHONPATH
diff --git a/sbin/spark-daemons.sh b/sbin/spark-daemons.sh
@@ -30,6 +30,24 @@ fi
 sbin=`dirname "$0"`
 sbin=`cd "$sbin"; pwd`
 
+# Check if --config is passed as an argument. It is an optional parameter.
+# Exit if the argument is not a directory.
+
+if [ "$1" == "--config" ]
+then
+  shift
+  conf_dir=$1
+  if [ ! -d "$conf_dir" ]
+  then
+    echo "ERROR : $conf_dir is not a directory"
+    echo $usage
+    exit 1
+  else
+    export SPARK_CONF_DIR=$conf_dir
+  fi
+  shift
+fi
+
 . "$sbin/spark-config.sh"
 
-exec "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/spark-daemon.sh" "$@"
+exec "$sbin/slaves.sh" --config $SPARK_CONF_DIR cd "$SPARK_HOME" \; "$sbin/spark-daemon.sh" --config $SPARK_CONF_DIR "$@"
diff --git a/sbin/start-slave.sh b/sbin/start-slave.sh
@@ -23,4 +23,27 @@
 sbin=`dirname "$0"`
 sbin=`cd "$sbin"; pwd`
 
-"$sbin"/spark-daemon.sh start org.apache.spark.deploy.worker.Worker "$@"
+# Check if --config is passed as an argument. It is an optional parameter.
+# Exit if the argument is not a directory.
+
+if [ "$1" == "--config" ]
+then
+  shift
+  conf_dir=$1
+  if [ ! -d "$conf_dir" ]
+  then
+    echo "ERROR : $conf_dir is not a directory"
+    echo $usage
+    exit 1
+  else
+    export SPARK_CONF_DIR=$conf_dir
+  fi
+  shift
+fi
+
+if [ "${SPARK_CONF_DIR}X" != "X" ]
+then
+    "$sbin"/spark-daemon.sh --config $SPARK_CONF_DIR start org.apache.spark.deploy.worker.Worker "$@"
+else
+    "$sbin"/spark-daemon.sh start org.apache.spark.deploy.worker.Worker "$@"
+fi
diff --git a/sbin/start-slaves.sh b/sbin/start-slaves.sh
@@ -58,12 +58,12 @@ fi
 
 # Launch the slaves
 if [ "$SPARK_WORKER_INSTANCES" = "" ]; then
-  exec "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/start-slave.sh" 1 spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT
+  exec "$sbin/slaves.sh" --config $SPARK_CONF_DIR cd "$SPARK_HOME" \; "$sbin/start-slave.sh" --config $SPARK_CONF_DIR 1 spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT
 else
   if [ "$SPARK_WORKER_WEBUI_PORT" = "" ]; then
     SPARK_WORKER_WEBUI_PORT=8081
   fi
   for ((i=0; i<$SPARK_WORKER_INSTANCES; i++)); do
-    "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/start-slave.sh" $(( $i + 1 ))  spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT --webui-port $(( $SPARK_WORKER_WEBUI_PORT + $i ))
+    "$sbin/slaves.sh" --config $SPARK_CONF_DIR cd "$SPARK_HOME" \; "$sbin/start-slave.sh" --config $SPARK_CONF_DIR $(( $i + 1 ))  spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT --webui-port $(( $SPARK_WORKER_WEBUI_PORT + $i ))
   done
 fi
diff --git a/sbin/stop-slaves.sh b/sbin/stop-slaves.sh
@@ -30,9 +30,9 @@ if [ -e "$sbin"/../tachyon/bin/tachyon ]; then
 fi
 
 if [ "$SPARK_WORKER_INSTANCES" = "" ]; then
-  "$sbin"/spark-daemons.sh stop org.apache.spark.deploy.worker.Worker 1
+  "$sbin"/spark-daemons.sh --config $SPARK_CONF_DIR stop org.apache.spark.deploy.worker.Worker 1
 else
   for ((i=0; i<$SPARK_WORKER_INSTANCES; i++)); do
-    "$sbin"/spark-daemons.sh stop org.apache.spark.deploy.worker.Worker $(( $i + 1 ))
+    "$sbin"/spark-daemons.sh --config $SPARK_CONF_DIR stop org.apache.spark.deploy.worker.Worker $(( $i + 1 ))
   done
 fi