diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 52355ab..80adecb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1386,9 +1386,10 @@ public static boolean isAclEnabled(Configuration conf) { public static final String NM_PRINCIPAL = NM_PREFIX + "principal"; - public static final String NM_AUX_SERVICES = + public static final String NM_AUX_SERVICES = NM_PREFIX + "aux-services"; - + + public static final String NM_AUX_SERVICE_FMT = NM_PREFIX + "aux-services.%s.class"; @@ -1398,6 +1399,9 @@ public static boolean isAclEnabled(Configuration conf) { public static final String NM_AUX_SERVICES_SYSTEM_CLASSES = NM_AUX_SERVICES + ".%s.system-classes"; + public static final String NM_AUX_SERVICES_BOOTSTRAP_PATH = + NM_AUX_SERVICES + ".%s.bootstrap-path"; + public static final String NM_USER_HOME_DIR = NM_PREFIX + "user-home-dir"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java index 171b20d..69f74d1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java @@ -18,10 +18,15 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager; +import java.io.File; +import java.io.IOException; +import java.net.URL; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.regex.Pattern; @@ -29,9 +34,14 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsUrlStreamHandlerFactory; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.Service; import org.apache.hadoop.service.ServiceStateChangeListener; @@ -58,6 +68,8 @@ private final Pattern p = Pattern.compile("^[A-Za-z_]+[A-Za-z0-9_]*$"); + private boolean setURLStreamHandler = false; + public AuxServices() { super(AuxServices.class.getName()); serviceMap = @@ -123,6 +135,8 @@ public void serviceInit(Configuration conf) throws Exception { String className = conf.get(classKey); final String appClassPath = conf.get(String.format( YarnConfiguration.NM_AUX_SERVICES_CLASSPATH, sName)); + String bootstrapPath = conf.get(String.format( + YarnConfiguration.NM_AUX_SERVICES_BOOTSTRAP_PATH, sName)); AuxiliaryService s = null; boolean useCustomerClassLoader = appClassPath != null && !appClassPath.isEmpty() && className != null @@ -132,6 +146,16 @@ public void serviceInit(Configuration conf) throws Exception { conf, className, appClassPath); LOG.info("The aux service:" + sName + " are using the custom classloader"); + } else if (bootstrapPath != null && !bootstrapPath.isEmpty()) { + if (!setURLStreamHandler) { + URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory()); + setURLStreamHandler = true; + } + URL[] urls = constructUrlsFromClasspath(conf, bootstrapPath); + s = AuxiliaryServiceWithCustomClassLoader.getInstance( + conf, className, urls); + LOG.info("The aux service:" + sName + + " are using the custom classloader."); } else { Class sClass = conf.getClass( classKey, null, AuxiliaryService.class); @@ -283,4 +307,34 @@ private void logWarningWhenAuxServiceThrowExceptions(AuxiliaryService service, : "The auxService name is " + service.getName()) + " and it got an error at event: " + eventType, th); } + + private URL[] constructUrlsFromClasspath(Configuration conf, + String classpath) throws AccessControlException, IOException { + FileContext fs = FileContext.getFileContext(conf); + List urls = new ArrayList(); + for (String element : classpath.split(File.pathSeparator)) { + if (element.endsWith("/*")) { + String dir = element.substring(0, element.length() - 1); + FileStatus[] files = fs.util().listStatus(new Path(dir), + new PathFilter() { + @Override + public boolean accept(Path path) { + return path.getName().endsWith(".jar") + || path.getName().endsWith(".JAR"); + } + }); + if (files != null) { + for (FileStatus file : files) { + urls.add(fs.makeQualified(file.getPath()).toUri().toURL()); + } + } + } else { + Path filePath = new Path(element); + if (fs.util().exists(filePath)) { + urls.add(fs.makeQualified(filePath).toUri().toURL()); + } + } + } + return urls.toArray(new URL[urls.size()]); + } } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxiliaryServiceWithCustomClassLoader.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxiliaryServiceWithCustomClassLoader.java index c764fbd..56ca0f9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxiliaryServiceWithCustomClassLoader.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxiliaryServiceWithCustomClassLoader.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.net.MalformedURLException; +import java.net.URL; import java.nio.ByteBuffer; import java.security.AccessController; import java.security.PrivilegedActionException; @@ -176,6 +177,24 @@ public static AuxiliaryServiceWithCustomClassLoader getInstance( customClassLoader); } + public static AuxiliaryServiceWithCustomClassLoader getInstance( + Configuration conf, String className, URL[] urls) + throws IOException, ClassNotFoundException { + String[] systemClasses = conf.getTrimmedStrings(String.format( + YarnConfiguration.NM_AUX_SERVICES_SYSTEM_CLASSES, + className)); + ClassLoader customClassLoader = createAuxServiceClassLoader( + urls, systemClasses); + Class clazz = Class.forName(className, true, + customClassLoader); + Class sClass = clazz.asSubclass( + AuxiliaryService.class); + AuxiliaryService wrapped = ReflectionUtils.newInstance(sClass, conf); + return new AuxiliaryServiceWithCustomClassLoader( + className + " with custom class loader", wrapped, + customClassLoader); + } + private static ClassLoader createAuxServiceClassLoader( final String appClasspath, final String[] systemClasses) throws IOException { @@ -198,4 +217,27 @@ public ClassLoader run() throws MalformedURLException { throw new IOException(e); } } + + private static ClassLoader createAuxServiceClassLoader( + final URL[] urls, final String[] systemClasses) + throws IOException { + try { + return AccessController.doPrivileged( + new PrivilegedExceptionAction() { + @Override + public ClassLoader run() throws MalformedURLException { + return new ApplicationClassLoader(urls, + AuxServices.class.getClassLoader(), + Arrays.asList(systemClasses)); + } + } + ); + } catch (PrivilegedActionException e) { + Throwable t = e.getCause(); + if (t instanceof MalformedURLException) { + throw (MalformedURLException) t; + } + throw new IOException(e); + } + } } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java index 26a1003..424a00e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java @@ -184,10 +184,11 @@ public ByteBuffer getMetaData() { } // To verify whether we could load class from customized class path. - // We would use ServiceC in this test. Also create a separate jar file - // including ServiceC class, and add this jar to customized directory. - // By setting some proper configurations, we should load ServiceC class - // from customized class path. + // We would use ServiceC in this test. Also create two separate jar files + // including ServiceC class, and add one jar to customized directory, and + // the other one to bootstrap directory. + // By setting some proper configurations, we can verify load ServiceC class + // from either customized class path or bootstrap classs path. @Test (timeout = 15000) public void testCustomizedAuxServiceClassPath() throws Exception { // verify that we can load AuxService Class from default Class path @@ -222,9 +223,13 @@ public void testCustomizedAuxServiceClassPath() throws Exception { rootDir.mkdirs(); } File testJar = null; + File testJarBootstrap = null; try { testJar = JarFinder.makeClassLoaderTestJar(this.getClass(), rootDir, "test-runjar.jar", 2048, ServiceC.class.getName()); + testJarBootstrap = JarFinder.makeClassLoaderTestJar(this.getClass(), + rootDir, "test-runjar-bootstrap.jar", 2048, + ServiceC.class.getName()); conf = new YarnConfiguration(); conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] {"ServiceC"}); @@ -233,6 +238,9 @@ public void testCustomizedAuxServiceClassPath() throws Exception { conf.set(String.format( YarnConfiguration.NM_AUX_SERVICES_CLASSPATH, "ServiceC"), testJar.getAbsolutePath()); + conf.set(String.format( + YarnConfiguration.NM_AUX_SERVICES_BOOTSTRAP_PATH, "ServiceC"), + testJarBootstrap.getAbsolutePath()); // remove "-org.apache.hadoop." from system classes String systemClasses = "-org.apache.hadoop." + "," + ApplicationClassLoader.SYSTEM_CLASSES_DEFAULT; @@ -244,6 +252,10 @@ public void testCustomizedAuxServiceClassPath() throws Exception { aux.start(); meta = aux.getMetaData(); Assert.assertTrue(meta.size() == 1); + // verify we load the auxService class in such sequences: + // 1. if we configured customized class path + // 2. if we configured bootstrap class path + // 3. NM local class path Set customizedAuxClassPath = null; for(Entry i : meta.entrySet()) { Assert.assertTrue(auxName.equals(i.getKey())); @@ -251,6 +263,7 @@ public void testCustomizedAuxServiceClassPath() throws Exception { customizedAuxClassPath = new HashSet(Arrays.asList(StringUtils .getTrimmedStrings(classPath))); Assert.assertTrue(classPath.contains(testJar.getName())); + Assert.assertFalse(classPath.contains(testJarBootstrap.getName())); } aux.stop(); @@ -259,6 +272,33 @@ public void testCustomizedAuxServiceClassPath() throws Exception { Set mutalClassPath = Sets.intersection(defaultAuxClassPath, customizedAuxClassPath); Assert.assertTrue(mutalClassPath.isEmpty()); + + // reset NM_AUX_SERVICES_CLASSPATH, and verify that we load + // auxService class from bootstrap class path + conf.set(String.format( + YarnConfiguration.NM_AUX_SERVICES_CLASSPATH, "ServiceC"), + ""); + aux = new AuxServices(); + aux.init(conf); + aux.start(); + meta = aux.getMetaData(); + Assert.assertTrue(meta.size() == 1); + Set bootstrapAuxClassPath = null; + for(Entry i : meta.entrySet()) { + Assert.assertTrue(auxName.equals(i.getKey())); + String classPath = Charsets.UTF_8.decode(i.getValue()).toString(); + bootstrapAuxClassPath = new HashSet(Arrays.asList(StringUtils + .getTrimmedStrings(classPath))); + Assert.assertTrue(classPath.contains(testJarBootstrap.getName())); + Assert.assertFalse(classPath.contains(testJar.getName())); + } + aux.stop(); + mutalClassPath = Sets.intersection(defaultAuxClassPath, + bootstrapAuxClassPath); + Assert.assertTrue(mutalClassPath.isEmpty()); + mutalClassPath = Sets.intersection(bootstrapAuxClassPath, + customizedAuxClassPath); + Assert.assertTrue(mutalClassPath.isEmpty()); } finally { if (testJar != null) { testJar.delete();