diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ce38d2762b2..e05219c2baf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1605,6 +1605,28 @@ public static boolean isAclEnabled(Configuration conf) {
public static final String NM_RESOURCE_PLUGINS =
NM_PREFIX + "resource-plugins";
+ /**
+ * This setting controls if pluggable device plugin framework is enabled.
+ * */
+ @Private
+ public static final String NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED =
+ NM_PREFIX + "pluggable-device-framework.enabled";
+
+ /**
+ * The pluggable device plugin framework is disabled by default
+ * */
+ @Private
+ public static final boolean DEFAULT_NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED =
+ false;
+
+ /**
+ * This settings contains vendor plugin class names for
+ * device plugin framework to load. Split by comma
+ * */
+ @Private
+ public static final String NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES =
+ NM_PREFIX + "pluggable-device-framework.device-classes";
+
/**
* Prefix for gpu configurations. Work in progress: This configuration
* parameter may be changed/removed in the future.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 1360e73f7fd..1671dcad970 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3770,6 +3770,25 @@
false
+
+
+ This settings controls if pluggable device framework is enabled.
+ Disabled by default
+
+ yarn.nodemanager.pluggable-device-framework.enabled
+ false
+
+
+
+
+ Configure vendor device plugin class name here. Comma separated.
+ The class must be found in CLASSPATH. The pluggable device framework will
+ load these classes.
+
+ yarn.nodemanager.pluggable-device-framework.device-classes
+
+
+
When yarn.nodemanager.resource.gpu.allowed-gpu-devices=auto specified,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
index f28aad206a6..e930d9665ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
@@ -23,6 +23,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaResourcePlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuResourcePlugin;
@@ -52,11 +53,10 @@
public synchronized void initialize(Context context)
throws YarnException {
Configuration conf = context.getConf();
- String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS);
+ Map pluginMap = new HashMap<>();
+ String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS);
if (plugins != null) {
- Map pluginMap = new HashMap<>();
-
// Initialize each plugins
for (String resourceName : plugins) {
resourceName = resourceName.trim();
@@ -92,8 +92,34 @@ public synchronized void initialize(Context context)
plugin.initialize(context);
pluginMap.put(resourceName, plugin);
}
+ }
+ // Try to load pluggable device plugins
+ boolean puggableDeviceFrameworkEnabled = conf.getBoolean(
+ YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+ YarnConfiguration.DEFAULT_NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED);
+
+ if (puggableDeviceFrameworkEnabled) {
+ initializePluggableDevicePlugins(context, conf, pluginMap);
+ } else {
+ LOG.info("The pluggable device framework is not enabled."
+ + " If you want, please set true to {}",
+ YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED);
+ }
+ configuredPlugins = Collections.unmodifiableMap(pluginMap);
+ }
- configuredPlugins = Collections.unmodifiableMap(pluginMap);
+ public void initializePluggableDevicePlugins(Context context,
+ Configuration configuration,
+ Map pluginMap)
+ throws YarnRuntimeException {
+ LOG.info("The pluggable device framework enabled," +
+ "trying to load the vendor plugins");
+
+ String[] pluginClassNames = configuration.getStrings(
+ YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES);
+ if (null == pluginClassNames) {
+ throw new YarnRuntimeException("Null value found in configuration: "
+ + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES);
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
index 6ed7c568899..2af89e836d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
@@ -24,6 +24,7 @@
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
@@ -41,27 +42,27 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.junit.After;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.*;
public class TestResourcePluginManager extends NodeManagerTestBase {
private NodeManager nm;
+ private YarnConfiguration conf;
+
+ @Before
+ public void setup() throws Exception {
+ this.conf = createNMConfig();
+ }
+
ResourcePluginManager stubResourcePluginmanager() {
// Stub ResourcePluginManager
final ResourcePluginManager rpm = mock(ResourcePluginManager.class);
@@ -145,7 +146,7 @@ public MyMockNM(ResourcePluginManager rpm) {
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
- ((NodeManager.NMContext)context).setResourcePluginManager(rpm);
+ ((NodeManager.NMContext) context).setResourcePluginManager(rpm);
return new BaseNodeStatusUpdaterForTest(context, dispatcher, healthChecker,
metrics, new BaseResourceTrackerForTest());
}
@@ -157,7 +158,7 @@ protected ContainerManagerImpl createContainerManager(Context context,
ApplicationACLsManager aclsManager,
LocalDirsHandlerService diskhandler) {
return new MyContainerManager(context, exec, del, nodeStatusUpdater,
- metrics, diskhandler);
+ metrics, diskhandler);
}
@Override
@@ -222,7 +223,7 @@ public void testLinuxContainerExecutorWithResourcePluginsEnabled() throws Except
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
- ((NMContext)context).setResourcePluginManager(rpm);
+ ((NMContext) context).setResourcePluginManager(rpm);
return new BaseNodeStatusUpdaterForTest(context, dispatcher, healthChecker,
metrics, new BaseResourceTrackerForTest());
}
@@ -239,7 +240,7 @@ protected ContainerManagerImpl createContainerManager(Context context,
@Override
protected ContainerExecutor createContainerExecutor(Configuration conf) {
- ((NMContext)this.getNMContext()).setResourcePluginManager(rpm);
+ ((NMContext) this.getNMContext()).setResourcePluginManager(rpm);
lce.setConf(conf);
return lce;
}
@@ -264,4 +265,89 @@ protected ContainerExecutor createContainerExecutor(Configuration conf) {
}
Assert.assertTrue("New ResourceHandler should be added", newHandlerAdded);
}
+
+ // Disabled pluggable framework in configuration.
+ // We use spy object of real rpm to verify "initializePluggableDevicePlugins"
+ // because use mock rpm will not working
+ @Test(timeout = 30000)
+ public void testInitializationWithPluggableDeviceFrameworkDisabled() throws Exception {
+ ResourcePluginManager rpm = new ResourcePluginManager();
+
+ ResourcePluginManager rpmSpy = spy(rpm);
+ nm = new MyMockNM(rpmSpy);
+
+ conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+ false);
+ nm.init(conf);
+ nm.start();
+ verify(rpmSpy, times(1)).initialize(
+ any(Context.class));
+ verify(rpmSpy, times(0)).initializePluggableDevicePlugins(
+ any(Context.class), any(Configuration.class), any(Map.class));
+ }
+
+ // No related configuration set.
+ @Test(timeout = 30000)
+ public void testInitializationWithPluggableDeviceFrameworkDisabled2()
+ throws Exception {
+ ResourcePluginManager rpm = new ResourcePluginManager();
+
+ ResourcePluginManager rpmSpy = spy(rpm);
+ nm = new MyMockNM(rpmSpy);
+
+ nm.init(conf);
+ nm.start();
+ verify(rpmSpy, times(1)).initialize(
+ any(Context.class));
+ verify(rpmSpy, times(0)).initializePluggableDevicePlugins(
+ any(Context.class), any(Configuration.class), any(Map.class));
+ }
+
+ // Enable framework and configure pluggable device classes
+ @Test(timeout = 30000)
+ public void testInitializationWithPluggableDeviceFrameworkEnabled()
+ throws Exception {
+ ResourcePluginManager rpm = new ResourcePluginManager();
+
+ ResourcePluginManager rpmSpy = spy(rpm);
+ nm = new MyMockNM(rpmSpy);
+
+ conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+ true);
+ conf.setStrings(
+ YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES,
+ "com.cmp1.hdw1plugin");
+ nm.init(conf);
+ nm.start();
+ verify(rpmSpy, times(1)).initialize(
+ any(Context.class));
+ verify(rpmSpy, times(1)).initializePluggableDevicePlugins(
+ any(Context.class), any(Configuration.class), any(Map.class));
+ }
+
+ // Enable pluggable framework, but leave device classes un-configured
+ // initializePluggableDevicePlugins invoked but it should throw an exception
+ @Test(timeout = 30000)
+ public void testInitializationWithPluggableDeviceFrameworkEnabled2() {
+ ResourcePluginManager rpm = new ResourcePluginManager();
+
+ ResourcePluginManager rpmSpy = spy(rpm);
+ nm = new MyMockNM(rpmSpy);
+ Boolean fail = false;
+ try {
+ YarnConfiguration conf = createNMConfig();
+ conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+ true);
+
+ nm.init(conf);
+ nm.start();
+ } catch (YarnRuntimeException e) {
+ fail = true;
+ } catch (Exception e) {
+
+ }
+ verify(rpmSpy, times(1)).initializePluggableDevicePlugins(
+ any(Context.class), any(Configuration.class), any(Map.class));
+ Assert.assertTrue(fail);
+ }
}