From c71377af421149a33445b5d93f2f7a2cc1ea7541 Mon Sep 17 00:00:00 2001 From: Andreina J Date: Fri, 5 Jun 2015 17:21:33 +0530 Subject: [PATCH] YARN-3771 --- .../main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java | 10 +++++++--- .../hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java | 2 +- .../java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 10 ++++++---- .../hadoop/yarn/applications/distributedshell/Client.java | 4 +++- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index c8d8a44..36f6258 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -205,11 +205,15 @@ private static void setMRFrameworkClasspath( // if the framework is specified then only use the MR classpath String frameworkName = getMRFrameworkName(conf); if (frameworkName == null) { + List defaultCrossPlatformClasspath = + YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH; + List defaultClasspath = + YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH; // Add standard Hadoop classes for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, - crossPlatform - ? YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH - : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { + crossPlatform ? defaultCrossPlatformClasspath + .toArray(new String[defaultCrossPlatformClasspath.size()]) + : defaultClasspath.toArray(new String[defaultClasspath.size()]))) { MRApps.addToEnvironment(environment, Environment.CLASSPATH.name(), c.trim(), conf); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java index d870d25..0a7cce9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java @@ -89,7 +89,7 @@ private void startCluster(Configuration conf) throws Exception { conf.set("hadoop.security.authentication", "simple"); String cp = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH, StringUtils.join(",", - YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) + YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) + File.pathSeparator + classpathDir; conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, cp); dfsCluster = new MiniDFSCluster.Builder(conf).build(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 72855cc..76547da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1301,7 +1301,8 @@ private static void addDeprecatedKeys() { */ @Public @Unstable - public static final String[] DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH= { + public static final List DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH= + Collections.unmodifiableList(Arrays.asList( ApplicationConstants.Environment.HADOOP_CONF_DIR.$$(), ApplicationConstants.Environment.HADOOP_COMMON_HOME.$$() + "/share/hadoop/common/*", @@ -1314,7 +1315,7 @@ private static void addDeprecatedKeys() { ApplicationConstants.Environment.HADOOP_YARN_HOME.$$() + "/share/hadoop/yarn/*", ApplicationConstants.Environment.HADOOP_YARN_HOME.$$() - + "/share/hadoop/yarn/lib/*" }; + + "/share/hadoop/yarn/lib/*" )); /** *

* Default platform-specific CLASSPATH for YARN applications. A @@ -1327,7 +1328,8 @@ private static void addDeprecatedKeys() { * a Linux/Unix server or vice versa. *

*/ - public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH = { + public static final List DEFAULT_YARN_APPLICATION_CLASSPATH = + Collections.unmodifiableList(Arrays.asList( ApplicationConstants.Environment.HADOOP_CONF_DIR.$(), ApplicationConstants.Environment.HADOOP_COMMON_HOME.$() + "/share/hadoop/common/*", @@ -1340,7 +1342,7 @@ private static void addDeprecatedKeys() { ApplicationConstants.Environment.HADOOP_YARN_HOME.$() + "/share/hadoop/yarn/*", ApplicationConstants.Environment.HADOOP_YARN_HOME.$() - + "/share/hadoop/yarn/lib/*" }; + + "/share/hadoop/yarn/lib/*")); /** Container temp directory */ public static final String DEFAULT_CONTAINER_TEMP_DIR = "./tmp"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 5a90880..0857dd4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -595,9 +595,11 @@ public boolean run() throws IOException, YarnException { // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$()) .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*"); + List defaultClasspath = + YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH; for (String c : conf.getStrings( YarnConfiguration.YARN_APPLICATION_CLASSPATH, - YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { + defaultClasspath.toArray(new String[defaultClasspath.size()]))) { classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR); classPathEnv.append(c.trim()); } -- 1.9.2.msysgit.0