diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index e3294d7..a8ec4bb 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -238,6 +238,11 @@ public static void setClasspath(Map environment, Configuration conf) throws IOException { boolean userClassesTakesPrecedence = conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false); + + if (userClassesTakesPrecedence) { + conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISCACHE, + "true"); + } String classpathEnvVar = conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 7ad8da8..cd480c3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -967,6 +967,16 @@ */ public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX + "application.classpath"; + + /** + * Whether or not entries from the distributed cache should be preferred over + * the rest of the YARN CLASSPATH + */ + public static final String YARN_APPLICATION_CLASSPATH_PREPEND_DISCACHE = + YARN_PREFIX + "application.classpath.prepend.distcache"; + + public static final boolean + DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISCACHE = false; /** * Default platform-agnostic CLASSPATH for YARN applications. A diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 0fd1200..adabbc6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -672,8 +672,28 @@ public void sanitizeEnv(Map environment, Path pwd, // additional testing. See YARN-358. if (Shell.WINDOWS) { String inputClassPath = environment.get(Environment.CLASSPATH.name()); + if (inputClassPath != null && !inputClassPath.isEmpty()) { - StringBuilder newClassPath = new StringBuilder(inputClassPath); + + //On non-windows, localized resources + //from distcache are available via the classpath as they were placed + //there but on windows they are not available when the classpath + //jar is created and so they "are lost" and have to be explicitly + //added to the classpath instead. This also means that their position + //is lost relative to other non-distcache classpath entries which will + //break things like mapreduce.job.user.classpath.first. + + boolean preferLocalizedJars = conf.getBoolean( + YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISCACHE, + YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISCACHE + ); + + boolean needsSeparator = false; + StringBuilder newClassPath = new StringBuilder(); + if (!preferLocalizedJars) { + newClassPath.append(inputClassPath); + needsSeparator = true; + } // Localized resources do not exist at the desired paths yet, because the // container launch script has not run to create symlinks yet. This @@ -687,7 +707,12 @@ public void sanitizeEnv(Map environment, Path pwd, for (String linkName : entry.getValue()) { // Append resource. - newClassPath.append(File.pathSeparator).append(pwd.toString()) + if (needsSeparator) { + newClassPath.append(File.pathSeparator); + } else { + needsSeparator = true; + } + newClassPath.append(pwd.toString()) .append(Path.SEPARATOR).append(linkName); // FileUtil.createJarWithClassPath must use File.toURI to convert @@ -704,6 +729,12 @@ public void sanitizeEnv(Map environment, Path pwd, } } } + if (preferLocalizedJars) { + if (needsSeparator) { + newClassPath.append(File.pathSeparator); + } + newClassPath.append(inputClassPath); + } // When the container launches, it takes the parent process's environment // and then adds/overwrites with the entries from the container launch