From bbbd907780cbd07507619bcc6d309e544e0c3471 Mon Sep 17 00:00:00 2001
From: KevinSmile <kevinwang013@hotmail.com>
Date: Sat, 12 Sep 2020 16:12:37 -0500
Subject: [PATCH] [SPARK-32804][LAUNCHER] Fix run-example command builder bug

### What changes were proposed in this pull request?

Bug fix in run-example command builder (as described in [SPARK-32804], run-example failed in standalone-cluster mode):
1. Missing primaryResource arg.
2. Wrong appResource arg.

which will affect `SparkSubmit` in Standalone-Cluster mode:
https://github.com/apache/spark/blob/32d87c2b595b4aac2d9274424a43697299638f61/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala#L695-L696

and get error at:
https://github.com/apache/spark/blob/f55694638d45f34ab91f6f6ec2066cbf7631f4af/core/src/main/scala/org/apache/spark/deploy/ClientArguments.scala#L74-L89

### Why are the changes needed?

Bug: run-example failed in standalone-cluster mode

### Does this PR introduce _any_ user-facing change?

Yes. User can run-example in standalone-cluster mode now.

### How was this patch tested?

New ut added.
Also it's a user-facing bug, so better re-check the real case in [SPARK-32804].

Closes #29653 from KevinSmile/bug-fix-master.

Authored-by: KevinSmile <kevinwang013@hotmail.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../launcher/SparkSubmitCommandBuilder.java    | 15 +++++++++++++--
 .../SparkSubmitCommandBuilderSuite.java        | 18 ++++++++++++++++++
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
index 383c3f60a595b..43e7f8debe17d 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
@@ -139,7 +139,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
 
         case RUN_EXAMPLE:
           isExample = true;
-          appResource = SparkLauncher.NO_RESOURCE;
+          appResource = findExamplesAppJar();
           submitArgs = args.subList(1, args.size());
       }
 
@@ -241,9 +241,11 @@ List<String> buildSparkSubmitArgs() {
     }
 
     args.addAll(parsedArgs);
+
     if (appResource != null) {
       args.add(appResource);
     }
+
     args.addAll(appArgs);
 
     return args;
@@ -401,6 +403,15 @@ private boolean isThriftServer(String mainClass) {
       mainClass.equals("org.apache.spark.sql.hive.thriftserver.HiveThriftServer2"));
   }
 
+  private String findExamplesAppJar() {
+    for (String exampleJar : findExamplesJars()) {
+      if (new File(exampleJar).getName().startsWith("spark-examples")) {
+        return exampleJar;
+      }
+    }
+    throw new IllegalStateException("Failed to find examples' main app jar.");
+  }
+
   private List<String> findExamplesJars() {
     boolean isTesting = "1".equals(getenv("SPARK_TESTING"));
     List<String> examplesJars = new ArrayList<>();
@@ -513,7 +524,7 @@ protected boolean handleUnknown(String opt) {
           className = EXAMPLE_CLASS_PREFIX + className;
         }
         mainClass = className;
-        appResource = SparkLauncher.NO_RESOURCE;
+        appResource = findExamplesAppJar();
         return false;
       } else if (errorOnUnknownArgs) {
         checkArgument(!opt.startsWith("-"), "Unrecognized option: %s", opt);
diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
index 752e8d4c23f8b..6cd089e256b93 100644
--- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
+++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
@@ -245,6 +245,24 @@ public void testExamplesRunner() throws Exception {
     assertEquals("42", cmd.get(cmd.size() - 1));
   }
 
+  @Test
+  public void testExamplesRunnerPrimaryResource() throws Exception {
+    List<String> sparkSubmitArgs = Arrays.asList(
+            SparkSubmitCommandBuilder.RUN_EXAMPLE,
+            parser.MASTER + "=foo",
+            parser.DEPLOY_MODE + "=cluster",
+            "SparkPi",
+            "100");
+
+    List<String> cmd = newCommandBuilder(sparkSubmitArgs).buildSparkSubmitArgs();
+    assertEquals(SparkSubmitCommandBuilder.EXAMPLE_CLASS_PREFIX + "SparkPi",
+            findArgValue(cmd, parser.CLASS));
+    assertEquals("cluster", findArgValue(cmd, parser.DEPLOY_MODE));
+    String primaryResource = cmd.get(cmd.size() - 2);
+    assertTrue(new File(primaryResource).getName().startsWith("spark-examples"));
+    assertFalse(cmd.contains(SparkLauncher.NO_RESOURCE));
+  }
+
   @Test(expected = IllegalArgumentException.class)
   public void testMissingAppResource() {
     new SparkSubmitCommandBuilder().buildSparkSubmitArgs();