Added Spark connect to the Spark master container

Signed-off-by: Norman Jordan <norman.jordan@improving.com>
opensearch-project · Jan 14, 2025 · 4c35a6c · 4c35a6c
1 parent 63e6b91
commit 4c35a6c
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 1 deletion.
diff --git a/docker/integ-test/docker-compose.yml b/docker/integ-test/docker-compose.yml
@@ -24,6 +24,7 @@ services:
       args:
         SPARK_VERSION: ${SPARK_VERSION:-3.5.3}
     container_name: spark
+    entrypoint: /opt/bitnami/scripts/spark/spark-master-entrypoint.sh
     ports:
       - "${MASTER_UI_PORT:-8080}:8080"
       - "${MASTER_PORT:-7077}:7077"

diff --git a/...test/opensearch/emr-src/org/opensearch/spark/emrserverless/DockerEMRServerlessClient.java b/...test/opensearch/emr-src/org/opensearch/spark/emrserverless/DockerEMRServerlessClient.java
@@ -131,8 +131,10 @@ public StartJobRunResult startJobRun(final StartJobRunRequest startJobRunRequest
     runContainerCmd.add("/opt/bitnami/spark/bin/spark-submit");
     runContainerCmd.add("--deploy-mode");
     runContainerCmd.add("client");
+    runContainerCmd.add("--exclude-packages");
+    runContainerCmd.add("org.opensearch:opensearch-spark-standalone_2.12,org.opensearch:opensearch-spark-sql-application_2.12,org.opensearch:opensearch-spark-ppl_2.12");
     runContainerCmd.add("--master");
-    runContainerCmd.add("spark://spark:7077");
+    runContainerCmd.add("local[2]");
 
     runContainerCmd.addAll(Arrays.asList(sparkSubmitParameters.split(" ")));
     runContainerCmd.addAll(entryPointArguments);

diff --git a/docker/integ-test/spark/Dockerfile b/docker/integ-test/spark/Dockerfile
@@ -11,3 +11,4 @@ RUN apt install -y curl
 USER 1001
 COPY ./spark-defaults.conf /opt/bitnami/spark/conf/spark-defaults.conf
 COPY ./log4j2.properties /opt/bitnami/spark/conf/log4j2.properties
+COPY ./spark-master-entrypoint.sh /opt/bitnami/scripts/spark/spark-master-entrypoint.sh
diff --git a/docker/integ-test/spark/spark-master-entrypoint.sh b/docker/integ-test/spark/spark-master-entrypoint.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+function start_spark_connect() {
+  sc_version=$(ls -1 /opt/bitnami/spark/jars/spark-core_*.jar | sed -e 's/^.*\/spark-core_//' -e 's/\.jar$//' -e 's/-/:/')
+
+  attempt=1
+  while [ -e "/tmp/spark_master_running" -a "$attempt" -le 10 ]; do
+    sleep 1
+    /opt/bitnami/spark/sbin/start-connect-server.sh --master spark://spark:7077 --packages org.apache.spark:spark-connect_${sc_version}
+    attempt=$(($attempt+1))
+  done
+}
+
+touch /tmp/spark_master_running
+start_spark_connect &
+/opt/bitnami/scripts/spark/entrypoint.sh /opt/bitnami/scripts/spark/run.sh
+rm /tmp/spark_master_running