Skip to content

Commit

Permalink
[SPARK-32804][Launcher] fix run-example command builder bug
Browse files Browse the repository at this point in the history
  • Loading branch information
KevinSmile committed Sep 7, 2020
1 parent f556946 commit 2c8c4bb
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {

case RUN_EXAMPLE:
isExample = true;
appResource = SparkLauncher.NO_RESOURCE;
appResource = findExamplesAppJar();
submitArgs = args.subList(1, args.size());
}

Expand Down Expand Up @@ -241,9 +241,11 @@ List<String> buildSparkSubmitArgs() {
}

args.addAll(parsedArgs);

if (appResource != null) {
args.add(appResource);
}

args.addAll(appArgs);

return args;
Expand Down Expand Up @@ -401,6 +403,15 @@ private boolean isThriftServer(String mainClass) {
mainClass.equals("org.apache.spark.sql.hive.thriftserver.HiveThriftServer2"));
}

private String findExamplesAppJar() {
for (String exampleJar : findExamplesJars()) {
if (new File(exampleJar).getName().startsWith("spark-examples")) {
return exampleJar;
}
}
throw new IllegalStateException("Failed to find examples' main app jar.");
}

private List<String> findExamplesJars() {
boolean isTesting = "1".equals(getenv("SPARK_TESTING"));
List<String> examplesJars = new ArrayList<>();
Expand Down Expand Up @@ -513,7 +524,7 @@ protected boolean handleUnknown(String opt) {
className = EXAMPLE_CLASS_PREFIX + className;
}
mainClass = className;
appResource = SparkLauncher.NO_RESOURCE;
appResource = findExamplesAppJar();
return false;
} else if (errorOnUnknownArgs) {
checkArgument(!opt.startsWith("-"), "Unrecognized option: %s", opt);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,24 @@ public void testExamplesRunner() throws Exception {
assertEquals("42", cmd.get(cmd.size() - 1));
}

@Test
public void testExamplesRunnerPrimaryResource() throws Exception {
List<String> sparkSubmitArgs = Arrays.asList(
SparkSubmitCommandBuilder.RUN_EXAMPLE,
parser.MASTER + "=foo",
parser.DEPLOY_MODE + "=cluster",
"SparkPi",
"100");

List<String> cmd = newCommandBuilder(sparkSubmitArgs).buildSparkSubmitArgs();
assertEquals(SparkSubmitCommandBuilder.EXAMPLE_CLASS_PREFIX + "SparkPi",
findArgValue(cmd, parser.CLASS));
assertEquals("cluster", findArgValue(cmd, parser.DEPLOY_MODE));
String primaryResource = cmd.get(cmd.size() - 2);
assertTrue(new File(primaryResource).getName().startsWith("spark-examples"));
assertFalse(cmd.contains(SparkLauncher.NO_RESOURCE));
}

@Test(expected = IllegalArgumentException.class)
public void testMissingAppResource() {
new SparkSubmitCommandBuilder().buildSparkSubmitArgs();
Expand Down

0 comments on commit 2c8c4bb

Please sign in to comment.