diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 4316a37..0a38dde 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1335,6 +1335,9 @@ private static void addDeprecatedKeys() { public static final String NM_AUX_SERVICE_FMT = NM_PREFIX + "aux-services.%s.class"; + public static final String NM_AUX_SERVICE_CLASS_LOADER_LOCATION = + NM_PREFIX + "aux-services.%s.class.classloader.location"; + public static final String NM_USER_HOME_DIR = NM_PREFIX + "user-home-dir"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java index cd5ed88..0cab5df 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager; +import java.net.URL; +import java.net.URLClassLoader; import java.nio.ByteBuffer; import java.util.Collection; import java.util.Collections; @@ -30,6 +32,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsUrlStreamHandlerFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.service.AbstractService; @@ -118,9 +121,21 @@ public void serviceInit(Configuration conf) throws Exception { YarnConfiguration.NM_AUX_SERVICES +" is invalid." + "The valid service name should only contain a-zA-Z0-9_ " + "and can not start with numbers"); - Class sClass = conf.getClass( - String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, sName), null, - AuxiliaryService.class); + String classKey = String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, sName); + Class sClass = null; + String className = conf.get(classKey); + String localClassPath = conf.get(String.format( + YarnConfiguration.NM_AUX_SERVICE_CLASS_LOADER_LOCATION, sName)); + if (localClassPath != null && !localClassPath.isEmpty() + && className != null && !className.isEmpty()) { + URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory()); + URL[] classLoaderUrls = new URL[]{new URL(localClassPath)}; + URLClassLoader urlClassLoader = new URLClassLoader(classLoaderUrls); + Class clazz = Class.forName(className, true, urlClassLoader); + sClass = clazz.asSubclass(AuxiliaryService.class); + } else { + sClass = conf.getClass(classKey, null,AuxiliaryService.class); + } if (null == sClass) { throw new RuntimeException("No class defined for " + sName); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java index 1380752..8521ea0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java @@ -28,6 +28,7 @@ import java.io.File; import java.io.IOException; +import java.net.URL; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; @@ -354,6 +355,56 @@ public void testAuxServiceRecoverySetup() throws IOException { } } + @Test (timeout = 10000) + public void testLoadAuxServiceLocally() throws IOException, Exception { + Configuration conf = new YarnConfiguration(); + conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, + new String[] { "Asrv"}); + conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"), + ServiceA.class, Service.class); + + // Set this configuration with a directory which does not + // contain the related jar file. + conf.set(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"), + TEST_DIR.toString()); + boolean auxServiceInit = true; + + // initiate the aux-service + // should be fail because the jar file can not be found in this configured + // directory. + try { + AuxServices aux = new AuxServices(); + aux.init(conf); + aux.close(); + } catch (Exception ex) { + auxServiceInit = false; + } finally { + FileUtil.fullyDelete(TEST_DIR); + } + Assert.assertFalse(auxServiceInit); + + // Set this configuration with the directory + // which contains the related jar file + ClassLoader loader = TestAuxServices.class.getClassLoader(); + String classPath = loader.getResource( + "org/apache/hadoop/yarn/server/nodemanager/containermanager" + + "/TestAuxServices.class").toString(); + Configuration conf2 = new YarnConfiguration(); + conf2.setStrings(YarnConfiguration.NM_AUX_SERVICES, + new String[] { "Asrv"}); + conf2.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"), + ServiceA.class, Service.class); + conf2.set(String.format(YarnConfiguration.NM_AUX_SERVICE_CLASS_LOADER_LOCATION, + "Asrv"), classPath); + + AuxServices aux = new AuxServices(); + aux.init(conf2); + Assert.assertEquals(1, aux.getServices().size()); + Assert.assertEquals(ServiceA.class.getName(), + (aux.getServices().toArray()[0]).getClass().getName()); + aux.close(); + } + static class RecoverableAuxService extends AuxiliaryService { static final FsPermission RECOVERY_PATH_PERMS = new FsPermission((short)0700);