Skip to content

Commit

Permalink
minor changes
Browse files Browse the repository at this point in the history
  • Loading branch information
mengxr committed May 21, 2014
1 parent 3e7e1c4 commit 11e5354
Showing 1 changed file with 7 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ trait ClientBase extends Logging {
}
}

var cachedJarLinks = ListBuffer.empty[String]
var cachedSecondaryJarLinks = ListBuffer.empty[String]
val fileLists = List( (args.addJars, LocalResourceType.FILE, true),
(args.files, LocalResourceType.FILE, false),
(args.archives, LocalResourceType.ARCHIVE, false) )
Expand All @@ -235,13 +235,13 @@ trait ClientBase extends Logging {
distCacheMgr.addResource(fs, conf, destPath, localResources, resType,
linkname, statCache)
if (addToClasspath) {
cachedJarLinks += linkname
cachedSecondaryJarLinks += linkname
}
}
}
}
}
sparkConf.set(ClientBase.CONF_SPARK_YARN_SECONDARY_JARS, cachedJarLinks.mkString(","))
sparkConf.set(ClientBase.CONF_SPARK_YARN_SECONDARY_JARS, cachedSecondaryJarLinks.mkString(","))

UserGroupInformation.getCurrentUser().addCredentials(credentials)
localResources
Expand Down Expand Up @@ -485,18 +485,19 @@ object ClientBase {

extraClassPath.foreach(addClasspathEntry)

val localSecondaryJarLinks = sparkConf.getOption(CONF_SPARK_YARN_SECONDARY_JARS).getOrElse("").split(",")
val cachedSecondaryJarLinks =
sparkConf.getOption(CONF_SPARK_YARN_SECONDARY_JARS).getOrElse("").split(",")
// Normally the users app.jar is last in case conflicts with spark jars
if (sparkConf.get("spark.yarn.user.classpath.first", "false").toBoolean) {
addPwdClasspathEntry(APP_JAR)
localSecondaryJarLinks.foreach(addPwdClasspathEntry)
cachedSecondaryJarLinks.foreach(addPwdClasspathEntry)
addPwdClasspathEntry(SPARK_JAR)
ClientBase.populateHadoopClasspath(conf, env)
} else {
addPwdClasspathEntry(SPARK_JAR)
ClientBase.populateHadoopClasspath(conf, env)
addPwdClasspathEntry(APP_JAR)
localSecondaryJarLinks.foreach(addPwdClasspathEntry)
cachedSecondaryJarLinks.foreach(addPwdClasspathEntry)
}
addPwdClasspathEntry("*")
}
Expand Down

0 comments on commit 11e5354

Please sign in to comment.