diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index 6d82104..ed9642b 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -242,8 +242,7 @@ public static void setClasspath(Map environment, conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false); if (userClassesTakesPrecedence) { - conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE, - "true"); + environment.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true"); } String classpathEnvVar = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index d42cd67..953c3ae 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -191,6 +191,11 @@ HADOOP_YARN_HOME("HADOOP_YARN_HOME"), /** + * $CLASSPATH_PREPEND_DISTCACHE + */ + CLASSPATH_PREPEND_DISTCACHE("CLASSPATH_PREPEND_DISTCACHE"), + + /** * $CONTAINER_ID * Final, exported by NodeManager and non-modifiable by users. */ diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 52fff14..835a618 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1259,16 +1259,6 @@ private static void addDeprecatedKeys() { + "application.classpath"; /** - * Whether or not entries from the distributed cache should be preferred over - * the rest of the YARN CLASSPATH - */ - public static final String YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE = - YARN_PREFIX + "application.classpath.prepend.distcache"; - - public static final boolean - DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE = false; - - /** * Default platform-agnostic CLASSPATH for YARN applications. A * comma-separated list of CLASSPATH entries. The parameter expansion marker * will be replaced with real parameter expansion marker ('%' for Windows and diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 8433c21..0cbcae5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -735,12 +735,17 @@ public void sanitizeEnv(Map environment, Path pwd, //jar is created and so they "are lost" and have to be explicitly //added to the classpath instead. This also means that their position //is lost relative to other non-distcache classpath entries which will - //break things like mapreduce.job.user.classpath.first. - - boolean preferLocalizedJars = conf.getBoolean( - YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE, - YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE - ); + //break things like mapreduce.job.user.classpath.first. An environment + //variable can be set to indicate that distcache entries should come + //first + + String classpathPrependDistCache = + environment.get(Environment.CLASSPATH_PREPEND_DISTCACHE.name()); + boolean preferLocalizedJars = false; + if (classpathPrependDistCache != null && + classpathPrependDistCache.equalsIgnoreCase("true")) { + preferLocalizedJars = true; + } boolean needsSeparator = false; StringBuilder newClassPath = new StringBuilder(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index efeeb1d..2856357 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -417,7 +417,7 @@ public void testPrependDistcache() throws Exception { userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME"); userSetEnv.put(Environment.PWD.name(), "user_set_PWD"); userSetEnv.put(Environment.HOME.name(), "user_set_HOME"); - userSetEnv.put(Environment.CLASSPATH.name(), "SYSTEM_CLPATH"); + userSetEnv.put(Environment.CLASSPATH.name(), "APATH"); containerLaunchContext.setEnvironment(userSetEnv); Container container = mock(Container.class); when(container.getContainerId()).thenReturn(cId); @@ -463,14 +463,12 @@ public void handle(Event event) { Assert.assertTrue( result.get(result.size() - 1).endsWith("userjarlink.jar")); - //Now move userjar to the front + //Then, with user classpath first + userSetEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true"); cId = ContainerId.newContainerId(appAttemptId, 1); when(container.getContainerId()).thenReturn(cId); - conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE, - "true"); - launch = new ContainerLaunch(distContext, conf, dispatcher, exec, null, container, dirsHandler, containerManager);