diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/Device.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/Device.java new file mode 100644 index 00000000000..295ee2fa701 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/Device.java @@ -0,0 +1,184 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin; + +import java.io.Serializable; +import java.util.Objects; + +public class Device implements Serializable, Comparable { + + private static final long serialVersionUID = 1L; + + /** + * Required fields: + * ID: an plugin specified index number + * devPath: device file like /dev/devicename + * majorNumber: major device number + * minorNumber: minor device number + * busID: PCI Bus ID in format [[[[]:]]:][][.[]]. Can get from "lspci -D" + * isHealthy: true or false indicating device health status + * */ + private final Integer ID; + private final String devPath; + private final Integer majorNumber; + private final Integer minorNumber; + private final String busID; + private boolean isHealthy; + + /** + * Optional fields + * */ + private String status; + // TODO: topology and attributes + + private Device(Builder builder) { + this.ID = Objects.requireNonNull(builder.ID); + this.devPath = Objects.requireNonNull(builder.devPath); + this.majorNumber = Objects.requireNonNull(builder.majorNumber); + this.minorNumber = Objects.requireNonNull(builder.minorNumber); + this.busID = Objects.requireNonNull(builder.busID); + this.isHealthy = Objects.requireNonNull(builder.isHealthy); + } + + public Integer getID() { + return ID; + } + + public String getDevPath() { + return devPath; + } + + public Integer getMajorNumber() { + return majorNumber; + } + + public Integer getMinorNumber() { + return minorNumber; + } + + public String getBusID() { + return busID; + } + + public boolean isHealthy() { + return isHealthy; + } + + public String getStatus() { + return status; + } + + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()){ + return false; + } + Device device = (Device) o; + return Objects.equals(ID, device.ID) && + Objects.equals(devPath, device.devPath) && + Objects.equals(majorNumber, device.majorNumber) && + Objects.equals(minorNumber, device.minorNumber) && + Objects.equals(busID, device.busID); + } + + @Override + public int hashCode() { + return Objects.hash(ID, devPath, majorNumber, minorNumber, busID); + } + + @Override + public int compareTo(Object o) { + if (o == null || (!(o instanceof Device))) { + return -1; + } + + Device other = (Device) o; + + int result = Integer.compare(ID, other.getID()); + if (0 != result) { + return result; + } + return Integer.compare(minorNumber, other.minorNumber); + } + + @Override + public String toString() { + return "(" + getDevPath() + ", " + getID() + ", " + getMajorNumber() + ":" + getMinorNumber() + ")"; + } + + public static class Builder { + private Integer ID; + private String devPath; + private Integer majorNumber; + private Integer minorNumber; + private String busID; + private boolean isHealthy; + private String status; + + private Builder() {} + + public static Builder newInstance() { + return new Builder(); + } + + public Device build(){ + return new Device(this); + } + + public Builder setID(Integer ID) { + this.ID = ID; + return this; + } + + public Builder setDevPath(String devPath) { + this.devPath = devPath; + return this; + } + + public Builder setMajorNumber(Integer majorNumber) { + this.majorNumber = majorNumber; + return this; + } + + public Builder setMinorNumber(Integer minorNumber) { + this.minorNumber = minorNumber; + return this; + } + + public Builder setBusID(String busID) { + this.busID = busID; + return this; + } + + public Builder setHealthy(boolean healthy) { + isHealthy = healthy; + return this; + } + + public Builder setStatus(String status) { + this.status = status; + return this; + } + + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DeviceFrameworkConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DeviceFrameworkConstants.java new file mode 100644 index 00000000000..4fbbe72b04d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DeviceFrameworkConstants.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin; + +public class DeviceFrameworkConstants { + /** + * Follows the "majorVersion.minorVersion.patchVersion" convention + * Given a version number MAJOR.MINOR.PATCH, increment the: + * + * MAJOR version when you make incompatible API changes, + * MINOR version when you add functionality in a backwards-compatible manner, and + * PATCH version when you make backwards-compatible bug fixes. + * */ + public static final String API_VERSION = "0.1.0"; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePlugin.java new file mode 100644 index 00000000000..746e5a9837d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePlugin.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin; + +import java.util.Set; + +/** + * Interface for vendor plugin to implement. + * */ +public interface DevicePlugin { + /** + * Called first when device plugin framework wants to register + * @return DeviceRegisterRequest {@link DeviceRegisterRequest} + * */ + DeviceRegisterRequest register(); + + /** + * Called when update node resource + * @return a set of {@link Device}, {@link java.util.TreeSet} recommended + * */ + Set getDevices(); + + /** + * Called before container launch. + * */ + DeviceRuntimeSpec preLaunchContainer(Set allocatedDevices); + + /** + * Called after container finish. + * */ + void postCompleteContainer(Set allocatedDevices); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DeviceRegisterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DeviceRegisterRequest.java new file mode 100644 index 00000000000..621ea20efa4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DeviceRegisterRequest.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin; + +import java.util.Objects; + +/** + * A vendor device plugin use this object to register + * to NM device plugin framework + * */ +public class DeviceRegisterRequest { + + // YARN device framework API apiVersion used in this plugin + private final String apiVersion; + + // plugin's own version + private final String pluginVersion; + private final String resourceName; + + public DeviceRegisterRequest(Builder builder) { + this.apiVersion = Objects.requireNonNull(builder.apiVersion); + this.resourceName = Objects.requireNonNull(builder.resourceName); + this.pluginVersion = builder.pluginVersion; + } + + public String getResourceName() { + return resourceName; + } + + public String getApiVersion() { + return apiVersion; + } + + public static class Builder { + private String apiVersion; + private String pluginVersion; + private String resourceName; + + private Builder() {} + + public static Builder newInstance() { + return new Builder(); + } + + public DeviceRegisterRequest build() { + return new DeviceRegisterRequest(this); + } + + // TODO: add sematic versioning pattern check + public Builder setApiVersion(String apiVersion) { + this.apiVersion = apiVersion; + return this; + } + + public Builder setResourceName(String resourceName) { + this.resourceName = resourceName; + return this; + } + + public Builder setPluginVersion(String pluginVersion) { + this.pluginVersion = pluginVersion; + return this; + } + + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DeviceRuntimeSpec.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DeviceRuntimeSpec.java new file mode 100644 index 00000000000..dc2fdf19e3e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DeviceRuntimeSpec.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin; + +public class DeviceRuntimeSpec { +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java index 69f15194fc2..a017e8976f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java @@ -18,15 +18,23 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceFrameworkConstants; +import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePlugin; +import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRegisterRequest; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework.DevicePluginAdapter; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaResourcePlugin; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuResourcePlugin; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,7 +59,7 @@ Collections.emptyMap(); public synchronized void initialize(Context context) - throws YarnException { + throws YarnException, ClassNotFoundException { Configuration conf = context.getConf(); Map pluginMap = new HashMap<>(); @@ -107,7 +115,8 @@ public synchronized void initialize(Context context) public void initializePluggableDevicePlugins(Context context, Configuration configuration, - Map pluginMap) throws YarnRuntimeException{ + Map pluginMap) + throws YarnRuntimeException, ClassNotFoundException { LOG.info("The pluggable device framework enabled, trying to load the vendor plugins"); String[] pluginClassNames = configuration.getStrings( YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES); @@ -115,6 +124,79 @@ public void initializePluggableDevicePlugins(Context context, throw new YarnRuntimeException("Null value found in configuration: " + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES); } + for (String pluginClassName : pluginClassNames) { + Class pluginClazz = Class.forName(pluginClassName); + if (!DevicePlugin.class.isAssignableFrom(pluginClazz)) { + throw new YarnRuntimeException("Class: " + pluginClassName + + " not instance of " + DevicePlugin.class.getCanonicalName()); + } + DevicePlugin dpInstance = (DevicePlugin) ReflectionUtils.newInstance(pluginClazz, + configuration); + // Try to register plugin + // TODO: handle the plugin method timeout issue + DeviceRegisterRequest request = dpInstance.register(); + // Check API version for compatibility + String apiVersion = request.getApiVersion(); + if (!isVersionCompatible(apiVersion, DeviceFrameworkConstants.API_VERSION)) { + throw new YarnRuntimeException("Class: " + pluginClassName + " API version: " + apiVersion + + " is not compatible. Expected: " + DeviceFrameworkConstants.API_VERSION); + } + String resourceName = request.getResourceName(); + // check if someone has already registered this resource type name + if (pluginMap.containsKey(resourceName)) { + throw new YarnRuntimeException(resourceName + + " has already been registered! Please change a resource type name"); + } + // check resource name is valid and configured in resource-types.xml + if (!isValidAndConfiguredResourceName(resourceName)) { + throw new YarnRuntimeException(resourceName + + " is not configured inside " + + YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE + + " , please configure it first"); + } + LOG.info("New resource type: " + resourceName + + " registered successfully by " + pluginClassName); + DevicePluginAdapter pluginAdapter = new DevicePluginAdapter(this, + resourceName, dpInstance); + LOG.info("Adapter of " + pluginClassName + " created. Initializing.."); + try { + pluginAdapter.initialize(context); + } catch (YarnException e) { + throw new YarnRuntimeException("Adapter of " + pluginClassName + " init failed!"); + } + LOG.info("Adapter of " + pluginClassName + " init success!"); + // Store plugin as adapter instance + pluginMap.put(request.getResourceName(), pluginAdapter); + } // end for + } + + @VisibleForTesting + public boolean isValidAndConfiguredResourceName(String resourceName) { + // check pattern match + // check configured + Map configuredResourceTypes = + ResourceUtils.getResourceTypes(); + if (!configuredResourceTypes.containsKey(resourceName)) { + return false; + } + return true; + } + + @VisibleForTesting + public boolean isVersionCompatible(String pluginApiVersion, + String frameworkApiVersion) { + // semantic version + String[] svs = pluginApiVersion.split("\\."); + String[] currentsvs = frameworkApiVersion.split("\\."); + // should be same major version + if (Integer.valueOf(svs[0]) != Integer.valueOf(currentsvs[0])) { + return false; + } + // should be older minor version + if (Integer.valueOf(svs[1]) > Integer.valueOf(currentsvs[1])) { + return false; + } + return true; } public synchronized void cleanup() throws YarnException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DevicePluginAdapter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DevicePluginAdapter.java new file mode 100644 index 00000000000..388c436b32a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DevicePluginAdapter.java @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device; +import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePlugin; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerVolumeCommand; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.DockerCommandPlugin; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo; + +import java.util.List; +import java.util.Set; + + +/** + * The {@link DevicePluginAdapter} will adapt existing hooks into vendor plugin's logic. + * It decouples the vendor plugin from YARN's device framework + * + * */ +public class DevicePluginAdapter extends NodeResourceUpdaterPlugin + implements ResourcePlugin, DockerCommandPlugin, ResourceHandler{ + final static Log LOG = LogFactory.getLog(DevicePluginAdapter.class); + + private ResourcePluginManager devicePluginManager; + private String resourceName; + private DevicePlugin devicePlugin; + private CGroupsHandler cGroupsHandler; + private PrivilegedOperationExecutor privilegedOperationExecutor; + + public DevicePluginAdapter(ResourcePluginManager pluginManager, String name, DevicePlugin dp) { + devicePluginManager = pluginManager; + resourceName = name; + devicePlugin = dp; + } + + /** + * Act as a {@link NodeResourceUpdaterPlugin} to update the {@link Resource} + * + * */ + @Override + public void updateConfiguredResource(Resource res) throws YarnException { + LOG.info(resourceName + " plugin update resource "); + Set devices = devicePlugin.getDevices(); + if (devices == null) { + LOG.warn(resourceName + " plugin failed to discover resource ( null value got)." ); + return; + } + res.setResourceValue(resourceName, devices.size()); + } + + /** + * Act as a {@link ResourcePlugin} + * */ + @Override + public void initialize(Context context) throws YarnException { + LOG.info(resourceName + " plugin adapter initialized"); + return; + } + + @Override + public ResourceHandler createResourceHandler(Context nmContext, CGroupsHandler cGroupsHandler, + PrivilegedOperationExecutor privilegedOperationExecutor) { + this.cGroupsHandler = cGroupsHandler; + this.privilegedOperationExecutor = privilegedOperationExecutor; + return this; + } + + @Override + public NodeResourceUpdaterPlugin getNodeResourceHandlerInstance() { + return this; + } + + @Override + public void cleanup() throws YarnException { + + } + + @Override + public DockerCommandPlugin getDockerCommandPluginInstance() { + return this; + } + + @Override + public NMResourceInfo getNMResourceInfo() throws YarnException { + return null; + } + + /** + * Act as a {@link DockerCommandPlugin} to hook the + * {@link org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime} + * */ + @Override + public void updateDockerRunCommand(DockerRunCommand dockerRunCommand, Container container) + throws ContainerExecutionException { + + } + + @Override + public DockerVolumeCommand getCreateDockerVolumeCommand(Container container) + throws ContainerExecutionException { + return null; + } + + @Override + public DockerVolumeCommand getCleanupDockerVolumesCommand(Container container) + throws ContainerExecutionException { + return null; + } + + /** + * Act as a {@link ResourceHandler} + * */ + @Override + public List bootstrap(Configuration configuration) throws ResourceHandlerException { + Set availableDevices = devicePlugin.getDevices(); + + /** + * We won't fail the NM if plugin returns invalid value here. + * // TODO: we should update RM's resource count if something wrong + * */ + if (availableDevices == null) { + LOG.error("Bootstrap " + resourceName + " failed. Null value got from plugin's getDevices method"); + return null; + } + + // TODO: Init cgroups + + return null; + } + + @Override + public List preStart(Container container) + throws ResourceHandlerException { + String containerIdStr = container.getContainerId().toString(); + /** + * TODO: implement a general container-executor device module to do isolation + * */ + return null; + } + + @Override + public List reacquireContainer(ContainerId containerId) + throws ResourceHandlerException { + return null; + } + + @Override + public List updateContainer(Container container) + throws ResourceHandlerException { + return null; + } + + @Override + public List postComplete(ContainerId containerId) + throws ResourceHandlerException { + return null; + } + + @Override + public List teardown() throws ResourceHandlerException { + return null; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/examples/FakeDevicePlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/examples/FakeDevicePlugin.java new file mode 100644 index 00000000000..a937efed78d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/examples/FakeDevicePlugin.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework.examples; + +import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.*; + +import java.util.Set; +import java.util.TreeSet; + +public class FakeDevicePlugin implements DevicePlugin { + @Override + public DeviceRegisterRequest register() { + return DeviceRegisterRequest.Builder.newInstance() + .setApiVersion(DeviceFrameworkConstants.API_VERSION) + .setResourceName("cmp.com/cmp") + .setPluginVersion("v1.0").build(); + } + + @Override + public Set getDevices() { + TreeSet r = new TreeSet<>(); + r.add(Device.Builder.newInstance() + .setID(0) + .setDevPath("/dev/cmp0") + .setMajorNumber(243) + .setMinorNumber(0) + .setBusID("0000:65:00.0") + .setHealthy(true) + .build()); + return r; + } + + @Override + public DeviceRuntimeSpec preLaunchContainer(Set allocatedDevices) { + return null; + } + + @Override + public void postCompleteContainer(Set allocatedDevices) { + + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java index 0333938ee8c..79e35ac2b7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.nodemanager.NodeManagerTestBase; import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater; +import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.*; import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; @@ -42,18 +43,21 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework.DevicePluginAdapter; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework.FakeTestDevicePlugin1; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework.FakeTestDevicePlugin2; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework.FakeTestDevicePlugin3; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework.examples.FakeDevicePlugin; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.apache.hadoop.yarn.util.resource.TestResourceUtils; import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; -import org.mockito.Mock; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.io.File; +import java.util.*; import static org.mockito.Matchers.any; import static org.mockito.Mockito.*; @@ -61,6 +65,19 @@ public class TestResourcePluginManager extends NodeManagerTestBase { private NodeManager nm; + private YarnConfiguration conf; + + private String tempResourceTypesFile; + + @Before + public void setup() throws Exception { + this.conf = createNMConfig(); + // setup resource-types.xml + ResourceUtils.resetResourceTypes(); + String resourceTypesFile = "resource-types-pluggable-devices.xml"; + this.tempResourceTypesFile = TestResourceUtils.setupResourceTypes(this.conf, resourceTypesFile); + } + ResourcePluginManager stubResourcePluginmanager() { // Stub ResourcePluginManager final ResourcePluginManager rpm = mock(ResourcePluginManager.class); @@ -93,6 +110,11 @@ public void tearDown() { // ignore } } + // cleanup resource-types.xml + File dest = new File(this.tempResourceTypesFile); + if (dest.exists()) { + dest.delete(); + } } private class CustomizedResourceHandler implements ResourceHandler { @@ -256,6 +278,9 @@ protected ContainerExecutor createContainerExecutor(Configuration conf) { boolean newHandlerAdded = false; for (ResourceHandler h : ((ResourceHandlerChain) handler) .getResourceHandlerList()) { + if (h instanceof DevicePluginAdapter) { + Assert.assertTrue(false); + } if (h instanceof CustomizedResourceHandler) { newHandlerAdded = true; break; @@ -274,7 +299,6 @@ public void testInitializationWithPluggableDeviceFrameworkDisabled() throws Exce ResourcePluginManager rpmSpy = spy(rpm); nm = new MyMockNM(rpmSpy); - YarnConfiguration conf = createNMConfig(); conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED, false); nm.init(conf); @@ -293,7 +317,6 @@ public void testInitializationWithPluggableDeviceFrameworkDisabled2() throws Exc ResourcePluginManager rpmSpy = spy(rpm); nm = new MyMockNM(rpmSpy); - YarnConfiguration conf = createNMConfig(); nm.init(conf); nm.start(); verify(rpmSpy, times(1)).initialize( @@ -310,11 +333,10 @@ public void testInitializationWithPluggableDeviceFrameworkEnabled() throws Excep ResourcePluginManager rpmSpy = spy(rpm); nm = new MyMockNM(rpmSpy); - YarnConfiguration conf = createNMConfig(); conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED, true); conf.setStrings(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES, - "com.cmp1.hdw1plugin"); + FakeDevicePlugin.class.getCanonicalName()); nm.init(conf); nm.start(); verify(rpmSpy, times(1)).initialize( @@ -326,7 +348,8 @@ public void testInitializationWithPluggableDeviceFrameworkEnabled() throws Excep // Enable pluggable framework, but leave device classes un-configured // initializePluggableDevicePlugins invoked but it should throw an exception @Test(timeout = 30000) - public void testInitializationWithPluggableDeviceFrameworkEnabled2() { + public void testInitializationWithPluggableDeviceFrameworkEnabled2() + throws ClassNotFoundException{ ResourcePluginManager rpm = new ResourcePluginManager(); ResourcePluginManager rpmSpy = spy(rpm); @@ -348,4 +371,108 @@ public void testInitializationWithPluggableDeviceFrameworkEnabled2() { any(Context.class), any(Configuration.class), any(Map.class)); Assert.assertTrue(fail); } + + @Test(timeout = 30000) + public void testNormalInitializationOfPluggableDeviceClasses() + throws Exception{ + + ResourcePluginManager rpm = new ResourcePluginManager(); + + ResourcePluginManager rpmSpy = spy(rpm); + nm = new MyMockNM(rpmSpy); + + conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED, + true); + conf.setStrings(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES, + FakeTestDevicePlugin1.class.getCanonicalName() + "," + + FakeDevicePlugin.class.getCanonicalName()); + nm.init(conf); + nm.start(); + Map pluginMap = rpmSpy.getNameToPlugins(); + Assert.assertEquals(2,pluginMap.size()); + ResourcePlugin rp = pluginMap.get("cmp.com/cmp"); + if (! (rp instanceof DevicePluginAdapter)) { + Assert.assertTrue(false); + } + } + + // Fail to load a class which doesn't implement interface DevicePlugin + @Test(timeout = 30000) + public void testLoadInvalidPluggableDeviceClasses() + throws Exception{ + ResourcePluginManager rpm = new ResourcePluginManager(); + + ResourcePluginManager rpmSpy = spy(rpm); + nm = new MyMockNM(rpmSpy); + + conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED, + true); + conf.setStrings(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES, + FakeTestDevicePlugin2.class.getCanonicalName()); + + String expectedMessage = "Class: " + FakeTestDevicePlugin2.class.getCanonicalName() + + " not instance of " + DevicePlugin.class.getCanonicalName(); + String actualMessage = ""; + try { + nm.init(conf); + nm.start(); + } catch (YarnRuntimeException e) { + actualMessage = e.getMessage(); + } + Assert.assertEquals(expectedMessage, actualMessage); + } + + @Test(timeout = 30000) + public void testLoadDuplicateResourceNameDevicePlugin() + throws Exception{ + ResourcePluginManager rpm = new ResourcePluginManager(); + + ResourcePluginManager rpmSpy = spy(rpm); + nm = new MyMockNM(rpmSpy); + + conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED, + true); + conf.setStrings(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES, + FakeTestDevicePlugin1.class.getCanonicalName() + "," + + FakeTestDevicePlugin3.class.getCanonicalName()); + + String expectedMessage = "cmpA.com/hdwA" + + " has already been registered! Please change a resource type name"; + String actualMessage = ""; + try { + nm.init(conf); + nm.start(); + } catch (YarnRuntimeException e) { + actualMessage = e.getMessage(); + } + Assert.assertEquals(expectedMessage, actualMessage); + } + + @Test(timeout = 30000) + public void testLoadInvalidApiVersionDevicePlugin() + throws Exception{ + ResourcePluginManager rpm = new ResourcePluginManager(); + String pluginApiVersion = "0.1.0"; + String frameworkApiVersion = "1.0.1"; + // major version is older than framework, incompatible + Boolean flag = rpm.isVersionCompatible(pluginApiVersion, frameworkApiVersion); + Assert.assertFalse(flag); + // same major version, different minor version. compatible + pluginApiVersion = "0.1.1"; + frameworkApiVersion = "0.2.0"; + flag = rpm.isVersionCompatible(pluginApiVersion, frameworkApiVersion); + Assert.assertTrue(flag); + } + + @Test(timeout = 30000) + public void testRequestedResourceNameIsConfigured() + throws Exception{ + ResourcePluginManager rpm = new ResourcePluginManager(); + String resourceName = "a.com/a"; + Assert.assertFalse(rpm.isValidAndConfiguredResourceName(resourceName)); + resourceName = "cmp.com/cmp"; + Assert.assertTrue(rpm.isValidAndConfiguredResourceName(resourceName)); + } + + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin1.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin1.java new file mode 100644 index 00000000000..262bfb5645d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin1.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework; + +import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.*; + +import java.util.Set; +import java.util.TreeSet; + +/** + * Used only for testing. + * A fake normal vendor plugin + * */ +public class FakeTestDevicePlugin1 implements DevicePlugin { + @Override + public DeviceRegisterRequest register() { + return DeviceRegisterRequest.Builder.newInstance() + .setApiVersion(DeviceFrameworkConstants.API_VERSION) + .setResourceName("cmpA.com/hdwA").build(); + } + + @Override + public Set getDevices() { + TreeSet r = new TreeSet<>(); + r.add(Device.Builder.newInstance() + .setID(0) + .setDevPath("/dev/hdwA0") + .setMajorNumber(243) + .setMinorNumber(0) + .setBusID("0000:65:00.0") + .setHealthy(true) + .build()); + return r; + } + + @Override + public DeviceRuntimeSpec preLaunchContainer(Set allocatedDevices) { + return null; + } + + @Override + public void postCompleteContainer(Set allocatedDevices) { + + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin2.java new file mode 100644 index 00000000000..8c1a61dc21c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin2.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework; + +/** + * Only used for testing. + * This isn't a implementation of DevicePlugin + * */ +public class FakeTestDevicePlugin2 { + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin3.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin3.java new file mode 100644 index 00000000000..c6419b184b3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin3.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework; + +import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.*; + +import java.util.Set; +import java.util.TreeSet; + +/** + * Only used for testing + * This plugin register a same name with FakeTestDevicePlugin1 + * */ +public class FakeTestDevicePlugin3 implements DevicePlugin { + @Override + public DeviceRegisterRequest register() { + return DeviceRegisterRequest.Builder.newInstance() + .setApiVersion(DeviceFrameworkConstants.API_VERSION) + .setResourceName("cmpA.com/hdwA").build(); + } + + @Override + public Set getDevices() { + TreeSet r = new TreeSet<>(); + r.add(Device.Builder.newInstance() + .setID(0) + .setDevPath("/dev/hdwA0") + .setMajorNumber(243) + .setMinorNumber(0) + .setBusID("0000:65:00.0") + .setHealthy(true) + .build()); + return r; + } + + @Override + public DeviceRuntimeSpec preLaunchContainer(Set allocatedDevices) { + return null; + } + + @Override + public void postCompleteContainer(Set allocatedDevices) { + + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/resource-types-pluggable-devices.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/resource-types-pluggable-devices.xml new file mode 100644 index 00000000000..65b8e752f1c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/resource-types-pluggable-devices.xml @@ -0,0 +1,7 @@ + + + + yarn.resource-types + cmp.com/cmp,cmpA.com/hdwA + + \ No newline at end of file