diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 4316a37..59c3719 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1335,6 +1335,9 @@ private static void addDeprecatedKeys() {
public static final String NM_AUX_SERVICE_FMT =
NM_PREFIX + "aux-services.%s.class";
+ public static final String NM_AUX_SERVICE_CLASS_LOADER_LOCATION =
+ NM_PREFIX + "aux-services.%s.class.classloader.location";
+
public static final String NM_USER_HOME_DIR =
NM_PREFIX + "user-home-dir";
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 49cced6..6eec783 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1725,6 +1725,18 @@
org.apache.hadoop.mapred.ShuffleHandler
+
+
+ The configuration to indicate the mapreduce_shuffle classpath location.
+ It can be configured as local file path (such as file:/path-to-class), or
+ as hdfs file path (such as hdfs:/path-to-class).
+ The mapreduce_shuffle class would be loaded using the configured classpath
+ instead of the default system class path.
+
+ yarn.nodemanager.aux-services.mapreduce_shuffle.class.classloader.location
+
+
+
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
index cd5ed88..670c634 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
@@ -18,7 +18,13 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
import java.nio.ByteBuffer;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -30,6 +36,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.service.AbstractService;
@@ -118,9 +125,20 @@ public void serviceInit(Configuration conf) throws Exception {
YarnConfiguration.NM_AUX_SERVICES +" is invalid." +
"The valid service name should only contain a-zA-Z0-9_ " +
"and can not start with numbers");
- Class extends AuxiliaryService> sClass = conf.getClass(
- String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, sName), null,
- AuxiliaryService.class);
+ String classKey =
+ String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, sName);
+ Class extends AuxiliaryService> sClass = null;
+ String className = conf.get(classKey);
+ String localClassPath = conf.get(String.format(
+ YarnConfiguration.NM_AUX_SERVICE_CLASS_LOADER_LOCATION, sName));
+ if (localClassPath != null && !localClassPath.isEmpty()
+ && className != null && !className.isEmpty()) {
+ URLClassLoader urlClassLoader = getClassLoader(localClassPath);
+ Class> clazz = Class.forName(className, true, urlClassLoader);
+ sClass = clazz.asSubclass(AuxiliaryService.class);
+ } else {
+ sClass = conf.getClass(classKey, null, AuxiliaryService.class);
+ }
if (null == sClass) {
throw new RuntimeException("No class defined for " + sName);
}
@@ -264,4 +282,17 @@ private void logWarningWhenAuxServiceThrowExceptions(AuxiliaryService service,
: "The auxService name is " + service.getName())
+ " and it got an error at event: " + eventType, th);
}
+
+ private URLClassLoader getClassLoader(final String localClassPath)
+ throws PrivilegedActionException{
+ return AccessController.doPrivileged(
+ new PrivilegedExceptionAction() {
+ @Override
+ public URLClassLoader run() throws MalformedURLException {
+ URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
+ URL[] classLoaderUrls = new URL[]{new URL(localClassPath)};
+ return new URLClassLoader(classLoaderUrls);
+ }
+ });
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
index 1380752..8521ea0 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
@@ -28,6 +28,7 @@
import java.io.File;
import java.io.IOException;
+import java.net.URL;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
@@ -354,6 +355,56 @@ public void testAuxServiceRecoverySetup() throws IOException {
}
}
+ @Test (timeout = 10000)
+ public void testLoadAuxServiceLocally() throws IOException, Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
+ new String[] { "Asrv"});
+ conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
+ ServiceA.class, Service.class);
+
+ // Set this configuration with a directory which does not
+ // contain the related jar file.
+ conf.set(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
+ TEST_DIR.toString());
+ boolean auxServiceInit = true;
+
+ // initiate the aux-service
+ // should be fail because the jar file can not be found in this configured
+ // directory.
+ try {
+ AuxServices aux = new AuxServices();
+ aux.init(conf);
+ aux.close();
+ } catch (Exception ex) {
+ auxServiceInit = false;
+ } finally {
+ FileUtil.fullyDelete(TEST_DIR);
+ }
+ Assert.assertFalse(auxServiceInit);
+
+ // Set this configuration with the directory
+ // which contains the related jar file
+ ClassLoader loader = TestAuxServices.class.getClassLoader();
+ String classPath = loader.getResource(
+ "org/apache/hadoop/yarn/server/nodemanager/containermanager"
+ + "/TestAuxServices.class").toString();
+ Configuration conf2 = new YarnConfiguration();
+ conf2.setStrings(YarnConfiguration.NM_AUX_SERVICES,
+ new String[] { "Asrv"});
+ conf2.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
+ ServiceA.class, Service.class);
+ conf2.set(String.format(YarnConfiguration.NM_AUX_SERVICE_CLASS_LOADER_LOCATION,
+ "Asrv"), classPath);
+
+ AuxServices aux = new AuxServices();
+ aux.init(conf2);
+ Assert.assertEquals(1, aux.getServices().size());
+ Assert.assertEquals(ServiceA.class.getName(),
+ (aux.getServices().toArray()[0]).getClass().getName());
+ aux.close();
+ }
+
static class RecoverableAuxService extends AuxiliaryService {
static final FsPermission RECOVERY_PATH_PERMS =
new FsPermission((short)0700);