diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ce38d2762b2..33256369b18 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1605,6 +1605,27 @@ public static boolean isAclEnabled(Configuration conf) { public static final String NM_RESOURCE_PLUGINS = NM_PREFIX + "resource-plugins"; + /** + * This setting controls if pluggable device plugin framework is enabled. + * */ + @Private + public static final String NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED = + NM_PREFIX + "pluggable-device-framework.enabled"; + + /** + * The pluggable device plugin framework is disabled by default + * */ + @Private + public static final boolean DEFAULT_NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED = false; + + /** + * This settings contains vendor plugin class names for device plugin framework to load. + * Split by comma + * */ + @Private + public static final String NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES = + NM_PREFIX + "pluggable-device-framework.device-classes"; + /** * Prefix for gpu configurations. Work in progress: This configuration * parameter may be changed/removed in the future. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 2ea6bfcc985..fbc46029a51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3770,6 +3770,25 @@ false + + + This settings controls if pluggable device framework is enabled. + Disabled by default + + yarn.nodemanager.pluggable-device-framework.enabled + false + + + + + Configure vendor device plugin class name here. Comma separated. + The class must be found in CLASSPATH. The pluggable device framework will + load these classes. + + yarn.nodemanager.pluggable-device-framework.device-classes + + + When yarn.nodemanager.resource.gpu.allowed-gpu-devices=auto specified, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java index f28aad206a6..69f15194fc2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaResourcePlugin; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuResourcePlugin; @@ -52,11 +53,10 @@ public synchronized void initialize(Context context) throws YarnException { Configuration conf = context.getConf(); - String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS); + Map pluginMap = new HashMap<>(); + String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS); if (plugins != null) { - Map pluginMap = new HashMap<>(); - // Initialize each plugins for (String resourceName : plugins) { resourceName = resourceName.trim(); @@ -92,8 +92,28 @@ public synchronized void initialize(Context context) plugin.initialize(context); pluginMap.put(resourceName, plugin); } + } + // Try to load pluggable device plugins + boolean puggableDeviceFrameworkEnabled = conf.getBoolean( + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED, + YarnConfiguration.DEFAULT_NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED); + if (puggableDeviceFrameworkEnabled) { + LOG.info("The pluggable device framework is not enabled. If you want, set true to " + + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED); + initializePluggableDevicePlugins(context, conf, pluginMap); + } + configuredPlugins = Collections.unmodifiableMap(pluginMap); + } - configuredPlugins = Collections.unmodifiableMap(pluginMap); + public void initializePluggableDevicePlugins(Context context, + Configuration configuration, + Map pluginMap) throws YarnRuntimeException{ + LOG.info("The pluggable device framework enabled, trying to load the vendor plugins"); + String[] pluginClassNames = configuration.getStrings( + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES); + if (null == pluginClassNames) { + throw new YarnRuntimeException("Null value found in configuration: " + + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java index 6ed7c568899..c5c60771bd3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; @@ -48,16 +49,14 @@ import org.junit.After; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mock; import java.util.HashMap; import java.util.List; import java.util.Map; import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; public class TestResourcePluginManager extends NodeManagerTestBase { private NodeManager nm; @@ -145,7 +144,7 @@ public MyMockNM(ResourcePluginManager rpm) { @Override protected NodeStatusUpdater createNodeStatusUpdater(Context context, Dispatcher dispatcher, NodeHealthCheckerService healthChecker) { - ((NodeManager.NMContext)context).setResourcePluginManager(rpm); + ((NodeManager.NMContext) context).setResourcePluginManager(rpm); return new BaseNodeStatusUpdaterForTest(context, dispatcher, healthChecker, metrics, new BaseResourceTrackerForTest()); } @@ -157,7 +156,7 @@ protected ContainerManagerImpl createContainerManager(Context context, ApplicationACLsManager aclsManager, LocalDirsHandlerService diskhandler) { return new MyContainerManager(context, exec, del, nodeStatusUpdater, - metrics, diskhandler); + metrics, diskhandler); } @Override @@ -222,7 +221,7 @@ public void testLinuxContainerExecutorWithResourcePluginsEnabled() throws Except @Override protected NodeStatusUpdater createNodeStatusUpdater(Context context, Dispatcher dispatcher, NodeHealthCheckerService healthChecker) { - ((NMContext)context).setResourcePluginManager(rpm); + ((NMContext) context).setResourcePluginManager(rpm); return new BaseNodeStatusUpdaterForTest(context, dispatcher, healthChecker, metrics, new BaseResourceTrackerForTest()); } @@ -239,7 +238,7 @@ protected ContainerManagerImpl createContainerManager(Context context, @Override protected ContainerExecutor createContainerExecutor(Configuration conf) { - ((NMContext)this.getNMContext()).setResourcePluginManager(rpm); + ((NMContext) this.getNMContext()).setResourcePluginManager(rpm); lce.setConf(conf); return lce; } @@ -264,4 +263,72 @@ protected ContainerExecutor createContainerExecutor(Configuration conf) { } Assert.assertTrue("New ResourceHandler should be added", newHandlerAdded); } + + // Disabled pluggable framework. + // We use spy object of real rpm to verify "initializePluggableDevicePlugins" + // because use mock rpm will not working + @Test(timeout = 30000) + public void testInitializationWithPluggableDeviceFrameworkDisabled() throws Exception { + ResourcePluginManager rpm = new ResourcePluginManager(); + + ResourcePluginManager rpmSpy = spy(rpm); + nm = new MyMockNM(rpmSpy); + + YarnConfiguration conf = createNMConfig(); + conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED, + false); + nm.init(conf); + nm.start(); + verify(rpmSpy, times(1)).initialize( + any(Context.class)); + verify(rpmSpy, times(0)).initializePluggableDevicePlugins( + any(Context.class), any(Configuration.class), any(Map.class)); + } + + // Enable framework and configure pluggable device classes + @Test(timeout = 30000) + public void testInitializationWithPluggableDeviceFrameworkEnabled() throws Exception { + ResourcePluginManager rpm = new ResourcePluginManager(); + + ResourcePluginManager rpmSpy = spy(rpm); + nm = new MyMockNM(rpmSpy); + + YarnConfiguration conf = createNMConfig(); + conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED, + true); + conf.setStrings(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES, + "com.cmp1.hdw1plugin"); + nm.init(conf); + nm.start(); + verify(rpmSpy, times(1)).initialize( + any(Context.class)); + verify(rpmSpy, times(1)).initializePluggableDevicePlugins( + any(Context.class), any(Configuration.class), any(Map.class)); + } + + // Enable pluggable framework, but leave device classes un-configured + // initializePluggableDevicePlugins invoked but it should throw an exception + @Test(timeout = 30000) + public void testInitializationWithPluggableDeviceFrameworkEnabled2() { + ResourcePluginManager rpm = new ResourcePluginManager(); + + ResourcePluginManager rpmSpy = spy(rpm); + nm = new MyMockNM(rpmSpy); + Boolean fail = false; + try { + YarnConfiguration conf = createNMConfig(); + conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED, + true); + + nm.init(conf); + nm.start(); + } catch (YarnRuntimeException e) { + fail = true; + } catch (Exception e) { + + } + verify(rpmSpy, times(1)).initializePluggableDevicePlugins( + any(Context.class), any(Configuration.class), any(Map.class)); + Assert.assertTrue(fail); + } }