Disallow --master with Spark Connect URL

apache · Feb 19, 2025 · 68a02ed · 68a02ed
1 parent 0af25b8
commit 68a02ed
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 0 deletions.
diff --git a/python/pyspark/errors/error-conditions.json b/python/pyspark/errors/error-conditions.json
@@ -512,6 +512,11 @@
       "A master URL must be set in your configuration."
     ]
   },
+  "MASTER_URL_INVALID": {
+    "message": [
+      "Master must either be yarn or start with spark, k8s, or local."
+    ]
+  },
   "MEMORY_PROFILE_INVALID_SOURCE":{
     "message": [
       "Memory profiler can only be used on editors with line numbers."

diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py
@@ -508,6 +508,11 @@ def getOrCreate(self) -> "SparkSession":
 
                             if url is None and is_api_mode_connect:
                                 url = opts.get("spark.master", os.environ.get("MASTER", "local"))
+                                if url.startswith("sc://"):
+                                    raise PySparkRuntimeError(
+                                        errorClass="MASTER_URL_INVALID",
+                                        messageParameters={},
+                                    )
 
                             if url is None:
                                 raise PySparkRuntimeError(