diff --git a/cli/bin/build-go.sh b/cli/bin/build-go.sh index 6cc43131..dc2380f7 100755 --- a/cli/bin/build-go.sh +++ b/cli/bin/build-go.sh @@ -34,7 +34,7 @@ export GOPATH=$(pwd)/vendor:$GOPATH # this may be omitted in 1.6+, left here for compatibility with 1.5: export GO15VENDOREXPERIMENT=1 - +go test # available GOOS/GOARCH permutations are listed at: # https://golang.org/doc/install/source#environment diff --git a/cli/dcos-spark/cli_test.go b/cli/dcos-spark/cli_test.go new file mode 100644 index 00000000..e57ba2a7 --- /dev/null +++ b/cli/dcos-spark/cli_test.go @@ -0,0 +1,39 @@ +package main + +import "testing" + +// test spaces +func TestCleanUpSubmitArgs(t *testing.T) { + _, args := sparkSubmitArgSetup() + inputArgs := "--conf spark.app.name=kerberosStreaming --conf spark.cores.max=8" + submitArgs, _ := cleanUpSubmitArgs(inputArgs, args.boolVals) + if "--conf=spark.app.name=kerberosStreaming" != submitArgs[0] { + t.Errorf("Failed to reduce spaces while cleaning submit args.") + } + + if "--conf=spark.cores.max=8" != submitArgs[1] { + t.Errorf("Failed to reduce spaces while cleaning submit args.") + } +} + +// test scopts pattern for app args when have full submit args +func TestScoptAppArgs(t *testing.T) { + _, args := sparkSubmitArgSetup() + inputArgs := `--driver-cores 1 --conf spark.cores.max=1 --driver-memory 512M + --class org.apache.spark.examples.SparkPi http://spark-example.jar --input1 value1 --input2 value2` + submitArgs, appFlags := cleanUpSubmitArgs(inputArgs, args.boolVals) + + if "--input1" != appFlags[0] { + t.Errorf("Failed to parse app args.") + } + if "value1" != appFlags[1] { + t.Errorf("Failed to parse app args.") + } + + if "--driver-memory=512M" != submitArgs[2] { + t.Errorf("Failed to parse submit args..") + } + if "http://spark-example.jar" != submitArgs[4] { + t.Errorf("Failed to parse submit args..") + } +} diff --git a/cli/dcos-spark/submit_builder.go b/cli/dcos-spark/submit_builder.go index 0f7692a7..f1a6c26b 100644 --- a/cli/dcos-spark/submit_builder.go +++ b/cli/dcos-spark/submit_builder.go @@ -20,6 +20,7 @@ import ( var keyWhitespaceValPattern = regexp.MustCompile("(.+)\\s+(.+)") var backslashNewlinePattern = regexp.MustCompile("\\s*\\\\s*\\n\\s+") +var collapseSpacesPattern = regexp.MustCompile(`[\s\p{Zs}]{2,}`) type sparkVal struct { flagName string @@ -298,8 +299,11 @@ func parseApplicationFile(args *sparkArgs) error { } func cleanUpSubmitArgs(argsStr string, boolVals []*sparkVal) ([]string, []string) { + + // collapse two or more spaces to one. + argsCompacted := collapseSpacesPattern.ReplaceAllString(argsStr, " ") // clean up any instances of shell-style escaped newlines: "arg1\\narg2" => "arg1 arg2" - argsCleaned := strings.TrimSpace(backslashNewlinePattern.ReplaceAllLiteralString(argsStr, " ")) + argsCleaned := strings.TrimSpace(backslashNewlinePattern.ReplaceAllLiteralString(argsCompacted, " ")) // HACK: spark-submit uses '--arg val' by convention, while kingpin only supports '--arg=val'. // translate the former into the latter for kingpin to parse. args := strings.Split(argsCleaned, " ")