diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index 6d82104..c8d8a44 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -241,11 +241,6 @@ public static void setClasspath(Map environment, boolean userClassesTakesPrecedence = conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false); - if (userClassesTakesPrecedence) { - conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE, - "true"); - } - String classpathEnvVar = conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false) ? Environment.APP_CLASSPATH.name() : Environment.CLASSPATH.name(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 52fff14..835a618 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1259,16 +1259,6 @@ private static void addDeprecatedKeys() { + "application.classpath"; /** - * Whether or not entries from the distributed cache should be preferred over - * the rest of the YARN CLASSPATH - */ - public static final String YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE = - YARN_PREFIX + "application.classpath.prepend.distcache"; - - public static final boolean - DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE = false; - - /** * Default platform-agnostic CLASSPATH for YARN applications. A * comma-separated list of CLASSPATH entries. The parameter expansion marker * will be replaced with real parameter expansion marker ('%' for Windows and diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 8433c21..41398b4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -735,12 +735,16 @@ public void sanitizeEnv(Map environment, Path pwd, //jar is created and so they "are lost" and have to be explicitly //added to the classpath instead. This also means that their position //is lost relative to other non-distcache classpath entries which will - //break things like mapreduce.job.user.classpath.first. - - boolean preferLocalizedJars = conf.getBoolean( - YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE, - YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE - ); + //break things like mapreduce.job.user.classpath.first. The relative + //location of the hadoop classpath entries and the job.jar entry + //indicates whether they should come before or after other entries, + //so this can be used to place them properly + + boolean preferLocalizedJars = false; + if (inputClassPath.toLowerCase().indexOf("job.jar") < + inputClassPath.toLowerCase().indexOf("hadoop")) { + preferLocalizedJars = true; + } boolean needsSeparator = false; StringBuilder newClassPath = new StringBuilder(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index efeeb1d..7814d72 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -417,7 +417,6 @@ public void testPrependDistcache() throws Exception { userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME"); userSetEnv.put(Environment.PWD.name(), "user_set_PWD"); userSetEnv.put(Environment.HOME.name(), "user_set_HOME"); - userSetEnv.put(Environment.CLASSPATH.name(), "SYSTEM_CLPATH"); containerLaunchContext.setEnvironment(userSetEnv); Container container = mock(Container.class); when(container.getContainerId()).thenReturn(cId); @@ -453,6 +452,9 @@ public void handle(Event event) { Path nmp = new Path(testDir); + //first, with system classpath (HADOOP) first + userSetEnv.put(Environment.CLASSPATH.name(), "HADOOP;JOB.JAR"); + launch.sanitizeEnv( userSetEnv, pwd, appDirs, containerLogs, resources, nmp); @@ -463,14 +465,12 @@ public void handle(Event event) { Assert.assertTrue( result.get(result.size() - 1).endsWith("userjarlink.jar")); - //Now move userjar to the front + //Then, with user classpath (JOB.JAR) first + userSetEnv.put(Environment.CLASSPATH.name(), "JOB.JAR;HADOOP"); cId = ContainerId.newContainerId(appAttemptId, 1); when(container.getContainerId()).thenReturn(cId); - conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE, - "true"); - launch = new ContainerLaunch(distContext, conf, dispatcher, exec, null, container, dirsHandler, containerManager);