diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 1824453..53c27d4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1351,6 +1351,9 @@ public static boolean isAclEnabled(Configuration conf) { public static final String NM_AUX_SERVICE_FMT = NM_PREFIX + "aux-services.%s.class"; + public static final String NM_AUX_SERVICE_CLASS_CLASSPATH = + NM_PREFIX + "aux-services.%s.class.classpath"; + public static final String NM_USER_HOME_DIR = NM_PREFIX + "user-home-dir"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index 529d63b..e403d19 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -132,6 +132,8 @@ public void initializeMemberVariables() { // Possibly obsolete, but unable to verify 100% xmlPropsToSkipCompare.add("yarn.nodemanager.aux-services.mapreduce_shuffle.class"); xmlPropsToSkipCompare.add("yarn.resourcemanager.container.liveness-monitor.interval-ms"); + xmlPropsToSkipCompare.add( + "yarn.nodemanager.aux-services.mapreduce_shuffle.class.classpath"); // Used in the XML file as a variable reference internal to the XML file xmlPropsToSkipCompare.add("yarn.nodemanager.hostname"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index d8ea3ad..ff99798 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1725,6 +1725,17 @@ org.apache.hadoop.mapred.ShuffleHandler + + + The configuration to indicate a comma-separated list of mapreduce_shuffle + local classpath location. + When this value is not empty, the mapreduce_shuffle class would be loaded + using the configured classpath, otherwise, it would use NM class path. + + yarn.nodemanager.aux-services.mapreduce_shuffle.class.classpath + + + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java index cd5ed88..355324b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java @@ -18,7 +18,11 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager; +import java.net.MalformedURLException; import java.nio.ByteBuffer; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -35,9 +39,11 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.Service; import org.apache.hadoop.service.ServiceStateChangeListener; +import org.apache.hadoop.util.ApplicationClassLoader; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext; import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext; import org.apache.hadoop.yarn.server.api.AuxiliaryService; @@ -118,21 +124,15 @@ public void serviceInit(Configuration conf) throws Exception { YarnConfiguration.NM_AUX_SERVICES +" is invalid." + "The valid service name should only contain a-zA-Z0-9_ " + "and can not start with numbers"); - Class sClass = conf.getClass( - String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, sName), null, - AuxiliaryService.class); - if (null == sClass) { - throw new RuntimeException("No class defined for " + sName); - } - AuxiliaryService s = ReflectionUtils.newInstance(sClass, conf); - // TODO better use s.getName()? - if(!sName.equals(s.getName())) { - LOG.warn("The Auxilurary Service named '"+sName+"' in the " - +"configuration is for "+sClass+" which has " - +"a name of '"+s.getName()+"'. Because these are " - +"not the same tools trying to send ServiceData and read " - +"Service Meta Data may have issues unless the refer to " - +"the name in the config."); + String classKey = String.format( + YarnConfiguration.NM_AUX_SERVICE_FMT, sName); + String className = conf.get(classKey); + final String localClassPath = conf.get(String.format( + YarnConfiguration.NM_AUX_SERVICE_CLASS_CLASSPATH, sName)); + AuxiliaryService s = createAuxiliaryService(localClassPath, conf, + sName, classKey, className); + if (null == s) { + throw new RuntimeException("No object created for " + sName); } addService(sName, s); if (recoveryEnabled) { @@ -264,4 +264,85 @@ private void logWarningWhenAuxServiceThrowExceptions(AuxiliaryService service, : "The auxService name is " + service.getName()) + " and it got an error at event: " + eventType, th); } -} + + AuxiliaryService callWithClassLoader(final String appClasspath, + Configuration conf, String serviceName, String classKey, + String className, Action action) + throws PrivilegedActionException { + // We load the (custom) classes; we make the custom classloader available + // and unset it once it is done + ClassLoader currentClassLoader = conf.getClassLoader(); + boolean useCustomerClassLoader = appClasspath != null + && !appClasspath.isEmpty() && className != null + && !className.isEmpty(); + if (useCustomerClassLoader) { + ClassLoader customClassLoader = AccessController.doPrivileged( + new PrivilegedExceptionAction() { + @Override + public ClassLoader run() throws MalformedURLException { + return new ApplicationClassLoader(appClasspath, + AuxServices.class.getClassLoader(), null); + } + }); + setClassLoader(customClassLoader, conf); + try { + return action.call(conf, customClassLoader, className); + } finally { + setClassLoader(currentClassLoader, conf); + } + } else { + Class sClass = conf.getClass( + classKey, null, AuxiliaryService.class); + + if (null == sClass) { + throw new RuntimeException("No class defined for " + serviceName); + } + AuxiliaryService s = ReflectionUtils.newInstance(sClass, conf); + // TODO better use s.getName()? + if(s != null && !serviceName.equals(s.getName())) { + LOG.warn("The Auxilurary Service named '"+serviceName+"' in the " + +"configuration is for "+sClass+" which has " + +"a name of '"+s.getName()+"'. Because these are " + +"not the same tools trying to send ServiceData and read " + +"Service Meta Data may have issues unless the refer to " + +"the name in the config."); + } + return s; + } + } + + private void setClassLoader(ClassLoader classLoader, + Configuration conf) { + if (classLoader != null) { + LOG.info("Setting classloader " + classLoader + + " on the configuration and as the thread context classloader"); + conf.setClassLoader(classLoader); + Thread.currentThread().setContextClassLoader(classLoader); + } + } + + private AuxiliaryService createAuxiliaryService(final String appClassPath, + Configuration conf, String serviceName, String classKey, + String className) throws PrivilegedActionException { + return callWithClassLoader(appClassPath, conf, serviceName, classKey, + className, new Action() { + public AuxiliaryService call(Configuration conf, + ClassLoader customClassLoader, String className) { + try { + Class clazz = Class.forName(className, true, + customClassLoader); + Class sClass = clazz.asSubclass( + AuxiliaryService.class); + return ReflectionUtils.newInstance(sClass, conf); + } catch (Exception e) { + throw new YarnRuntimeException(e); + } + } + }); + } + + private static interface Action { + T call(Configuration conf, ClassLoader customClassLoader, + String className); + } +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java index 1380752..d70771f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java @@ -354,6 +354,68 @@ public void testAuxServiceRecoverySetup() throws IOException { } } + @Test (timeout = 10000) + public void testLoadAuxServiceLocally() throws IOException, Exception { + Configuration conf = new YarnConfiguration(); + conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, + new String[] { "Asrv"}); + conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"), + ServiceA.class, Service.class); + + // Set this configuration with a directory which does not + // contain the related jar file. + conf.set(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"), + TEST_DIR.toString()); + boolean auxServiceInit = true; + + // initiate the aux-service + // should be fail because the jar file can not be found in this configured + // directory. + AuxServices aux = null; + try { + aux = new AuxServices(); + aux.init(conf); + } catch (Exception ex) { + auxServiceInit = false; + } finally { + if (aux != null) { + aux.close(); + } + } + Assert.assertFalse("The aux-service should not be initiated" + + "by specifing the invalid classpath.", + auxServiceInit); + + // Set this configuration with the directory + // which contains the related jar file + ClassLoader loader = TestAuxServices.class.getClassLoader(); + String classPath = loader.getResource( + "org/apache/hadoop/yarn/server/nodemanager/containermanager" + + "/TestAuxServices.class").toString(); + Configuration conf2 = new YarnConfiguration(); + conf2.setStrings(YarnConfiguration.NM_AUX_SERVICES, + new String[] { "Asrv"}); + conf2.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"), + ServiceA.class, Service.class); + conf2.set(String.format(YarnConfiguration.NM_AUX_SERVICE_CLASS_CLASSPATH, + "Asrv"), classPath); + + AuxServices aux1 = null; + try { + aux1 = new AuxServices(); + aux1.init(conf2); + Assert.assertEquals("We should only have one aux-service initiated.", + 1, aux1.getServices().size()); + Assert.assertEquals("ServiceA should be initiated as NM Aux-service.", + ServiceA.class.getName(), + (aux1.getServices().toArray()[0]).getClass().getName()); + } finally { + if (aux1 != null) { + aux1.close(); + } + } + } + static class RecoverableAuxService extends AuxiliaryService { static final FsPermission RECOVERY_PATH_PERMS = new FsPermission((short)0700);