diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ce38d2762b2..33256369b18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1605,6 +1605,27 @@ public static boolean isAclEnabled(Configuration conf) {
public static final String NM_RESOURCE_PLUGINS =
NM_PREFIX + "resource-plugins";
+ /**
+ * This setting controls if pluggable device plugin framework is enabled.
+ * */
+ @Private
+ public static final String NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED =
+ NM_PREFIX + "pluggable-device-framework.enabled";
+
+ /**
+ * The pluggable device plugin framework is disabled by default
+ * */
+ @Private
+ public static final boolean DEFAULT_NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED = false;
+
+ /**
+ * This settings contains vendor plugin class names for device plugin framework to load.
+ * Split by comma
+ * */
+ @Private
+ public static final String NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES =
+ NM_PREFIX + "pluggable-device-framework.device-classes";
+
/**
* Prefix for gpu configurations. Work in progress: This configuration
* parameter may be changed/removed in the future.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 2ea6bfcc985..fbc46029a51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3770,6 +3770,25 @@
false
+
+
+ This settings controls if pluggable device framework is enabled.
+ Disabled by default
+
+ yarn.nodemanager.pluggable-device-framework.enabled
+ false
+
+
+
+
+ Configure vendor device plugin class name here. Comma separated.
+ The class must be found in CLASSPATH. The pluggable device framework will
+ load these classes.
+
+ yarn.nodemanager.pluggable-device-framework.device-classes
+
+
+
When yarn.nodemanager.resource.gpu.allowed-gpu-devices=auto specified,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
index f28aad206a6..69f15194fc2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
@@ -23,6 +23,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaResourcePlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuResourcePlugin;
@@ -52,11 +53,10 @@
public synchronized void initialize(Context context)
throws YarnException {
Configuration conf = context.getConf();
- String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS);
+ Map pluginMap = new HashMap<>();
+ String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS);
if (plugins != null) {
- Map pluginMap = new HashMap<>();
-
// Initialize each plugins
for (String resourceName : plugins) {
resourceName = resourceName.trim();
@@ -92,8 +92,28 @@ public synchronized void initialize(Context context)
plugin.initialize(context);
pluginMap.put(resourceName, plugin);
}
+ }
+ // Try to load pluggable device plugins
+ boolean puggableDeviceFrameworkEnabled = conf.getBoolean(
+ YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+ YarnConfiguration.DEFAULT_NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED);
+ if (puggableDeviceFrameworkEnabled) {
+ LOG.info("The pluggable device framework is not enabled. If you want, set true to " +
+ YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED);
+ initializePluggableDevicePlugins(context, conf, pluginMap);
+ }
+ configuredPlugins = Collections.unmodifiableMap(pluginMap);
+ }
- configuredPlugins = Collections.unmodifiableMap(pluginMap);
+ public void initializePluggableDevicePlugins(Context context,
+ Configuration configuration,
+ Map pluginMap) throws YarnRuntimeException{
+ LOG.info("The pluggable device framework enabled, trying to load the vendor plugins");
+ String[] pluginClassNames = configuration.getStrings(
+ YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES);
+ if (null == pluginClassNames) {
+ throw new YarnRuntimeException("Null value found in configuration: " +
+ YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES);
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
index 6ed7c568899..c5c60771bd3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
@@ -24,6 +24,7 @@
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
@@ -48,16 +49,14 @@
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.Mock;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.*;
public class TestResourcePluginManager extends NodeManagerTestBase {
private NodeManager nm;
@@ -145,7 +144,7 @@ public MyMockNM(ResourcePluginManager rpm) {
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
- ((NodeManager.NMContext)context).setResourcePluginManager(rpm);
+ ((NodeManager.NMContext) context).setResourcePluginManager(rpm);
return new BaseNodeStatusUpdaterForTest(context, dispatcher, healthChecker,
metrics, new BaseResourceTrackerForTest());
}
@@ -157,7 +156,7 @@ protected ContainerManagerImpl createContainerManager(Context context,
ApplicationACLsManager aclsManager,
LocalDirsHandlerService diskhandler) {
return new MyContainerManager(context, exec, del, nodeStatusUpdater,
- metrics, diskhandler);
+ metrics, diskhandler);
}
@Override
@@ -222,7 +221,7 @@ public void testLinuxContainerExecutorWithResourcePluginsEnabled() throws Except
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
- ((NMContext)context).setResourcePluginManager(rpm);
+ ((NMContext) context).setResourcePluginManager(rpm);
return new BaseNodeStatusUpdaterForTest(context, dispatcher, healthChecker,
metrics, new BaseResourceTrackerForTest());
}
@@ -239,7 +238,7 @@ protected ContainerManagerImpl createContainerManager(Context context,
@Override
protected ContainerExecutor createContainerExecutor(Configuration conf) {
- ((NMContext)this.getNMContext()).setResourcePluginManager(rpm);
+ ((NMContext) this.getNMContext()).setResourcePluginManager(rpm);
lce.setConf(conf);
return lce;
}
@@ -264,4 +263,72 @@ protected ContainerExecutor createContainerExecutor(Configuration conf) {
}
Assert.assertTrue("New ResourceHandler should be added", newHandlerAdded);
}
+
+ // Disabled pluggable framework.
+ // We use spy object of real rpm to verify "initializePluggableDevicePlugins"
+ // because use mock rpm will not working
+ @Test(timeout = 30000)
+ public void testInitializationWithPluggableDeviceFrameworkDisabled() throws Exception {
+ ResourcePluginManager rpm = new ResourcePluginManager();
+
+ ResourcePluginManager rpmSpy = spy(rpm);
+ nm = new MyMockNM(rpmSpy);
+
+ YarnConfiguration conf = createNMConfig();
+ conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+ false);
+ nm.init(conf);
+ nm.start();
+ verify(rpmSpy, times(1)).initialize(
+ any(Context.class));
+ verify(rpmSpy, times(0)).initializePluggableDevicePlugins(
+ any(Context.class), any(Configuration.class), any(Map.class));
+ }
+
+ // Enable framework and configure pluggable device classes
+ @Test(timeout = 30000)
+ public void testInitializationWithPluggableDeviceFrameworkEnabled() throws Exception {
+ ResourcePluginManager rpm = new ResourcePluginManager();
+
+ ResourcePluginManager rpmSpy = spy(rpm);
+ nm = new MyMockNM(rpmSpy);
+
+ YarnConfiguration conf = createNMConfig();
+ conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+ true);
+ conf.setStrings(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES,
+ "com.cmp1.hdw1plugin");
+ nm.init(conf);
+ nm.start();
+ verify(rpmSpy, times(1)).initialize(
+ any(Context.class));
+ verify(rpmSpy, times(1)).initializePluggableDevicePlugins(
+ any(Context.class), any(Configuration.class), any(Map.class));
+ }
+
+ // Enable pluggable framework, but leave device classes un-configured
+ // initializePluggableDevicePlugins invoked but it should throw an exception
+ @Test(timeout = 30000)
+ public void testInitializationWithPluggableDeviceFrameworkEnabled2() {
+ ResourcePluginManager rpm = new ResourcePluginManager();
+
+ ResourcePluginManager rpmSpy = spy(rpm);
+ nm = new MyMockNM(rpmSpy);
+ Boolean fail = false;
+ try {
+ YarnConfiguration conf = createNMConfig();
+ conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+ true);
+
+ nm.init(conf);
+ nm.start();
+ } catch (YarnRuntimeException e) {
+ fail = true;
+ } catch (Exception e) {
+
+ }
+ verify(rpmSpy, times(1)).initializePluggableDevicePlugins(
+ any(Context.class), any(Configuration.class), any(Map.class));
+ Assert.assertTrue(fail);
+ }
}