diff --git hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 45aa868830c..e6dcefb2099 100644 --- hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -633,4 +633,12 @@ + + + + + + + + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 7e8c01d3636..92137addb05 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -18,8 +18,12 @@ package org.apache.hadoop.yarn.api.records; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import com.google.common.collect.Lists; +import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.NotImplementedException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Public; @@ -212,6 +216,22 @@ public void setMemorySize(long memory) { return resources; } + /** + * Get list of resource information, this will be used by JAXB. + * @return list of resources copy. + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + public List getAllResourcesListCopy() { + List list = new ArrayList<>(); + for (ResourceInformation i : resources) { + ResourceInformation ri = new ResourceInformation(); + ResourceInformation.copy(i, ri); + list.add(ri); + } + return list; + } + /** * Get ResourceInformation for a specified resource. * diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java index f4a49f93025..493aa7b7578 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java @@ -30,11 +30,13 @@ import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -224,7 +226,20 @@ public synchronized void cleanupAssignGpus(ContainerId containerId) { } @VisibleForTesting - public synchronized Map getDeviceAllocationMapping() { - return new HashMap<>(usedDevices); + public synchronized Map getDeviceAllocationMappingCopy() { + return new HashMap<>(usedDevices); + } + + public synchronized List getAllowedGpusCopy() { + return new ArrayList<>(allowedGpuDevices); + } + + public synchronized List getAssignedGpusCopy() { + List assigns = new ArrayList<>(); + for (Map.Entry entry : usedDevices.entrySet()) { + assigns.add(new AssignedGpuDevice(entry.getKey().getIndex(), + entry.getKey().getMinorNumber(), entry.getValue())); + } + return assigns; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java index 4a783d312f6..500382162fb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java @@ -133,7 +133,6 @@ public GpuResourceHandlerImpl(Context nmContext, return ret; } - @VisibleForTesting public GpuResourceAllocator getGpuAllocator() { return gpuAllocator; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java index 6e134b34efb..78167c4ef33 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo; /** * {@link ResourcePlugin} is an interface for node manager to easier support @@ -80,4 +81,14 @@ ResourceHandler createResourceHandler(Context nmContext, * @throws YarnException if any issue occurs */ void cleanup() throws YarnException; + + /** + * Get resource information from this plugin. + * + * @return NMResourceInfo, an example is + * {@link org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation} + * + * @throws YarnException when any issue occurs + */ + NMResourceInfo getNMResourceInfo() throws YarnException; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/AssignedGpuDevice.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/AssignedGpuDevice.java new file mode 100644 index 00000000000..df4b905d185 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/AssignedGpuDevice.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.yarn.api.records.ContainerId; + +/** + * In addition to {@link GpuDevice}, this include container id and more runtime + * information related to who is using the GPU device if possible + */ +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class AssignedGpuDevice extends GpuDevice { + private static final long serialVersionUID = -12983712986315L; + + String containerId; + + public AssignedGpuDevice() { + + } + + public AssignedGpuDevice(int index, int minorNumber, + ContainerId containerId) { + super(index, minorNumber); + this.containerId = containerId.toString(); + } + + public String getContainerId() { + return containerId; + } + + public void setContainerId(String containerId) { + this.containerId = containerId; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || !(obj instanceof AssignedGpuDevice)) { + return false; + } + AssignedGpuDevice other = (AssignedGpuDevice) obj; + return index == other.index && minorNumber == other.minorNumber + && containerId.equals(other.containerId); + } + + @Override + public int compareTo(Object obj) { + if (obj == null || (!(obj instanceof AssignedGpuDevice))) { + return -1; + } + + AssignedGpuDevice other = (AssignedGpuDevice) obj; + + int result = Integer.compare(index, other.index); + if (0 != result) { + return result; + } + result = Integer.compare(minorNumber, other.minorNumber); + if (0 != result) { + return result; + } + return containerId.compareTo(other.containerId); + } + + @Override + public int hashCode() { + final int prime = 47; + return prime * (prime * index + minorNumber) + containerId.hashCode(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java index 811992416d0..6f084e635b4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java @@ -19,15 +19,25 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu; import java.io.Serializable; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + /** * This class is used to represent GPU device while allocation. */ +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) public class GpuDevice implements Serializable, Comparable { - private int index; - private int minorNumber; + protected int index; + protected int minorNumber; private static final long serialVersionUID = -6812314470754667710L; + public GpuDevice() { + + } + public GpuDevice(int index, int minorNumber) { this.index = index; this.minorNumber = minorNumber; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java index 9576ce7fec2..d294503704e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java @@ -18,17 +18,25 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu.GpuResourceAllocator; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu.GpuResourceHandlerImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.NMGpuResourceInfo; + +import java.util.List; +import java.util.Map; public class GpuResourcePlugin implements ResourcePlugin { - private ResourceHandler gpuResourceHandler = null; + private GpuResourceHandlerImpl gpuResourceHandler = null; private GpuNodeResourceUpdateHandler resourceDiscoverHandler = null; @Override @@ -58,4 +66,18 @@ public synchronized NodeResourceUpdaterPlugin getNodeResourceHandlerInstance() { public void cleanup() throws YarnException { // Do nothing. } + + @Override + public NMResourceInfo getNMResourceInfo() throws YarnException { + GpuDeviceInformation gpuDeviceInformation = + GpuDiscoverer.getInstance().getGpuDeviceInformation(); + GpuResourceAllocator gpuResourceAllocator = + gpuResourceHandler.getGpuAllocator(); + List totalGpus = gpuResourceAllocator.getAllowedGpusCopy(); + List assignedGpuDevices = + gpuResourceAllocator.getAssignedGpusCopy(); + + return new NMGpuResourceInfo(gpuDeviceInformation, totalGpus, + assignedGpuDevices); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 60905d7bf45..7476d759edc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -27,6 +27,10 @@ import java.util.List; import java.util.Map.Entry; import java.util.Set; + +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -488,6 +492,29 @@ public void write(OutputStream os) throws IOException, } } + @GET + @Path("/resources/{resourcename}") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Object getNMResourceInfo( + @PathParam("resourcename") + String resourceName) throws YarnException { + init(); + ResourcePluginManager rpm = this.nmContext.getResourcePluginManager(); + if (rpm != null && rpm.getNameToPlugins() != null) { + ResourcePlugin plugin = rpm.getNameToPlugins().get(resourceName); + if (plugin != null) { + NMResourceInfo nmResourceInfo = plugin.getNMResourceInfo(); + if (nmResourceInfo != null) { + return nmResourceInfo; + } + } + } + + throw new YarnException( + "Could not get detailed resource information for given resource-name=" + + resourceName); + } + private long parseLongParam(String bytes) { if (bytes == null || bytes.isEmpty()) { return Long.MAX_VALUE; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMResourceInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMResourceInfo.java new file mode 100644 index 00000000000..18ce8ea7a68 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMResourceInfo.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class NMResourceInfo { +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java index 977032a3487..837d5cc99cd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java @@ -25,7 +25,7 @@ import java.util.List; /** - * All GPU Device Information in the system. + * All GPU Device Information in the system, fetched from nvidia-smi. */ @InterfaceAudience.Private @InterfaceStability.Unstable diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/NMGpuResourceInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/NMGpuResourceInfo.java new file mode 100644 index 00000000000..e5855374314 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/NMGpuResourceInfo.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo; + +import java.util.List; + +/** + * Gpu device information return to client when + * {@link org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices#getNMResourceInfo(String)} + * is invoked. + */ +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class NMGpuResourceInfo extends NMResourceInfo { + GpuDeviceInformation gpuDeviceInformation; + + List totalGpuDevices; + List assignedGpuDevices; + + public NMGpuResourceInfo() { + + } + + public NMGpuResourceInfo(GpuDeviceInformation gpuDeviceInformation, + List totalGpuDevices, + List assignedGpuDevices) { + this.gpuDeviceInformation = gpuDeviceInformation; + this.totalGpuDevices = totalGpuDevices; + this.assignedGpuDevices = assignedGpuDevices; + } + + public GpuDeviceInformation getGpuDeviceInformation() { + return gpuDeviceInformation; + } + + public void setGpuDeviceInformation( + GpuDeviceInformation gpuDeviceInformation) { + this.gpuDeviceInformation = gpuDeviceInformation; + } + + public List getTotalGpuDevices() { + return totalGpuDevices; + } + + public void setTotalGpuDevices(List totalGpuDevices) { + this.totalGpuDevices = totalGpuDevices; + } + + public List getAssignedGpuDevices() { + return assignedGpuDevices; + } + + public void setAssignedGpuDevices( + List assignedGpuDevices) { + this.assignedGpuDevices = assignedGpuDevices; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java index f3153136b6a..25c2e3a1f1d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java @@ -135,7 +135,7 @@ public void setGpuUtilizations(PerGpuUtilizations utilizations) { this.gpuUtilizations = utilizations; } - @XmlElement(name = "bar1_memory_usage") + @XmlElement(name = "fb_memory_usage") public PerGpuMemoryUsage getGpuMemoryUsage() { return gpuMemoryUsage; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java index 3964c4e415e..afc1a9679b7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java @@ -27,7 +27,7 @@ @InterfaceAudience.Private @InterfaceStability.Unstable -@XmlRootElement(name = "bar1_memory_usage") +@XmlRootElement(name = "fb_memory_usage") public class PerGpuMemoryUsage { long usedMemoryMiB = -1L; long availMemoryMiB = -1L; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java index d985b5b6fb9..b5796dfced2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java @@ -374,7 +374,7 @@ public void testRecoverResourceAllocation() throws Exception { gpuResourceHandler.reacquireContainer(getContainerId(1)); Map deviceAllocationMapping = - gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping(); + gpuResourceHandler.getGpuAllocator().getDeviceAllocationMappingCopy(); Assert.assertEquals(2, deviceAllocationMapping.size()); Assert.assertTrue( deviceAllocationMapping.keySet().contains(new GpuDevice(1, 1))); @@ -408,7 +408,7 @@ public void testRecoverResourceAllocation() throws Exception { // Make sure internal state not changed. deviceAllocationMapping = - gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping(); + gpuResourceHandler.getGpuAllocator().getDeviceAllocationMappingCopy(); Assert.assertEquals(2, deviceAllocationMapping.size()); Assert.assertTrue(deviceAllocationMapping.keySet() .containsAll(Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 3)))); @@ -440,7 +440,7 @@ public void testRecoverResourceAllocation() throws Exception { // Make sure internal state not changed. deviceAllocationMapping = - gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping(); + gpuResourceHandler.getGpuAllocator().getDeviceAllocationMappingCopy(); Assert.assertEquals(2, deviceAllocationMapping.size()); Assert.assertTrue(deviceAllocationMapping.keySet() .containsAll(Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 3)))); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index 4586a7b88c4..72071dae1f9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -18,25 +18,20 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringReader; -import java.net.HttpURLConnection; -import java.net.URI; -import java.net.URL; -import java.util.List; -import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.core.MediaType; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - -import org.junit.Assert; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.servlet.GuiceServletContextListener; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.ClientResponse.Status; +import com.sun.jersey.api.client.GenericType; +import com.sun.jersey.api.client.UniformInterfaceException; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.WebAppDescriptor; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; @@ -48,6 +43,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo; import org.apache.hadoop.yarn.logaggregation.TestContainerLogsUtils; @@ -59,7 +55,15 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice; import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.NMGpuResourceInfo; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.PerGpuDeviceInformation; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams; @@ -73,6 +77,7 @@ import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.w3c.dom.Document; @@ -80,24 +85,35 @@ import org.w3c.dom.NodeList; import org.xml.sax.InputSource; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.servlet.GuiceServletContextListener; -import com.google.inject.servlet.ServletModule; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.ClientResponse.Status; -import com.sun.jersey.api.client.GenericType; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; -import com.sun.jersey.test.framework.WebAppDescriptor; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.MediaType; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringReader; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * Test the nodemanager node info web services api's */ public class TestNMWebServices extends JerseyTestBase { - private static Context nmContext; + private static NodeManager.NMContext nmContext; private static ResourceView resourceView; private static ApplicationACLsManager aclsManager; private static LocalDirsHandlerService dirsHandler; @@ -411,6 +427,116 @@ public void testNMRedirect() { assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID)); } + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + private static class MockNMResourceInfo extends NMResourceInfo { + public long a = 1000L; + public MockNMResourceInfo() { } + } + + @Test + public void testGetNMResourceInfo() + throws YarnException, InterruptedException, JSONException { + ResourcePluginManager rpm = mock(ResourcePluginManager.class); + Map namesToPlugins = new HashMap<>(); + ResourcePlugin mockPlugin1 = mock(ResourcePlugin.class); + NMResourceInfo nmResourceInfo1 = new MockNMResourceInfo(); + when(mockPlugin1.getNMResourceInfo()).thenReturn(nmResourceInfo1); + namesToPlugins.put("resource-1", mockPlugin1); + namesToPlugins.put("yarn.io/resource-1", mockPlugin1); + ResourcePlugin mockPlugin2 = mock(ResourcePlugin.class); + namesToPlugins.put("resource-2", mockPlugin2); + when(rpm.getNameToPlugins()).thenReturn(namesToPlugins); + + nmContext.setResourcePluginManager(rpm); + + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("node").path( + "resources").path("resource-2").accept(MediaType.APPLICATION_JSON).get( + ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON, response.getType().toString()); + + // Access resource-2 should fail (null NMResourceInfo returned). + JSONObject json = response.getEntity(JSONObject.class); + assertIncludesException(json); + + // Access resource-3 should fail (unkown plugin) + response = r.path("ws").path("v1").path("node").path( + "resources").path("resource-3").accept(MediaType.APPLICATION_JSON).get( + ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON, response.getType().toString()); + json = response.getEntity(JSONObject.class); + assertIncludesException(json); + + // Access resource-1 should success + response = r.path("ws").path("v1").path("node").path( + "resources").path("resource-1").accept(MediaType.APPLICATION_JSON).get( + ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON, response.getType().toString()); + json = response.getEntity(JSONObject.class); + assertEquals(1000, Long.parseLong(json.get("a").toString())); + + // Access resource-1 should success (encoded yarn.io/Fresource-1). + response = r.path("ws").path("v1").path("node").path("resources").path( + "yarn.io%2Fresource-1").accept(MediaType.APPLICATION_JSON).get( + ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON, response.getType().toString()); + json = response.getEntity(JSONObject.class); + assertEquals(1000, Long.parseLong(json.get("a").toString())); + } + + private ContainerId createContainerId(int id) { + ApplicationId appId = ApplicationId.newInstance(0, 0); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + ContainerId containerId = ContainerId.newContainerId(appAttemptId, id); + return containerId; + } + + @Test + public void testGetYarnGpuResourceInfo() + throws YarnException, InterruptedException, JSONException { + ResourcePluginManager rpm = mock(ResourcePluginManager.class); + Map namesToPlugins = new HashMap<>(); + ResourcePlugin mockPlugin1 = mock(ResourcePlugin.class); + GpuDeviceInformation gpuDeviceInformation = new GpuDeviceInformation(); + gpuDeviceInformation.setDriverVersion("1.2.3"); + gpuDeviceInformation.setGpus(Arrays.asList(new PerGpuDeviceInformation())); + NMResourceInfo nmResourceInfo1 = new NMGpuResourceInfo(gpuDeviceInformation, + Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 2), + new GpuDevice(3, 3)), Arrays + .asList(new AssignedGpuDevice(2, 2, createContainerId(1)), + new AssignedGpuDevice(3, 3, createContainerId(2)))); + when(mockPlugin1.getNMResourceInfo()).thenReturn(nmResourceInfo1); + namesToPlugins.put("resource-1", mockPlugin1); + namesToPlugins.put("yarn.io/resource-1", mockPlugin1); + ResourcePlugin mockPlugin2 = mock(ResourcePlugin.class); + namesToPlugins.put("resource-2", mockPlugin2); + when(rpm.getNameToPlugins()).thenReturn(namesToPlugins); + + nmContext.setResourcePluginManager(rpm); + + WebResource r = resource(); + ClientResponse response; + JSONObject json; + + // Access resource-1 should success + response = r.path("ws").path("v1").path("node").path( + "resources").path("resource-1").accept(MediaType.APPLICATION_JSON).get( + ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON, response.getType().toString()); + json = response.getEntity(JSONObject.class); + assertEquals("1.2.3", + json.getJSONObject("gpuDeviceInformation").get("driver_version")); + assertEquals(3, json.getJSONArray("totalGpuDevices").length()); + assertEquals(2, json.getJSONArray("assignedGpuDevices").length()); + assertEquals(2, json.getJSONArray("assignedGpuDevices").length()); + } + + private void assertIncludesException(JSONObject json) { + assertTrue(json.has("RemoteException")); + } + private void testContainerLogs(WebResource r, ContainerId containerId) throws IOException { final String containerIdStr = containerId.toString(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java index e22597d5881..dc96746cf5d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java @@ -39,7 +39,7 @@ public void testParse() throws IOException, YarnException { Assert.assertEquals(2, info.getGpus().size()); PerGpuDeviceInformation gpu1 = info.getGpus().get(1); Assert.assertEquals("Tesla P100-PCIE-12GB", gpu1.getProductName()); - Assert.assertEquals(16384, gpu1.getGpuMemoryUsage().getTotalMemoryMiB()); + Assert.assertEquals(12193, gpu1.getGpuMemoryUsage().getTotalMemoryMiB()); Assert.assertEquals(10.3f, gpu1.getGpuUtilizations().getOverallGpuUtilization(), 1e-6); Assert.assertEquals(34f, gpu1.getTemperature().getCurrentGpuTemp(), 1e-6); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-nm-gpu.js hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-nm-gpu.js new file mode 100644 index 00000000000..bf6307a664c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-nm-gpu.js @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AbstractAdapter from './abstract'; + +export default AbstractAdapter.extend({ + + address: "localBaseAddress", + restNameSpace: "node", + serverName: "NM", + + urlForFindRecord(id/*, modelName, snapshot*/) { + var url = this._buildURL(); + url = url.replace("{nodeAddress}", id) + "/resources/yarn.io%2Fgpu"; + return url; + } + +}); \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js index ce268117fc6..5236ca05571 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js @@ -20,6 +20,7 @@ import Ember from 'ember'; import BaseChartComponent from 'yarn-ui/components/base-chart-component'; import ColorUtils from 'yarn-ui/utils/color-utils'; import Converter from 'yarn-ui/utils/converter'; +import {Entities} from 'yarn-ui/constants'; export default BaseChartComponent.extend({ /* @@ -41,8 +42,10 @@ export default BaseChartComponent.extend({ } if (!middleValue) { - if (this.get("type") === "memory") { + if (this.get(Entities.Type) === Entities.Memory) { middleValue = Converter.memoryToSimpliedUnit(total); + } else if (this.get(Entities.Type) === Entities.Resource) { + middleValue = Converter.resourceToSimplifiedUnit(total, this.get(Entities.Unit)); } else { middleValue = total; } @@ -151,7 +154,10 @@ export default BaseChartComponent.extend({ var value = d.value; if (this.get("type") === "memory") { value = Converter.memoryToSimpliedUnit(value); + } else if (this.get("type") === "resource") { + value = Converter.resourceToSimplifiedUnit(value, this.get(Entities.Unit)); } + return d.label + ' = ' + value + suffix; }.bind(this)); } @@ -185,10 +191,18 @@ export default BaseChartComponent.extend({ } this.renderDonutChart(this.get("data"), this.get("title"), this.get("showLabels"), - this.get("middleLabel"), this.get("middleValue")); + this.get("middleLabel"), this.get("middleValue"), this.get("suffix")); }, didInsertElement: function() { + // When parentIdPrefix is specified, use parentidPrefix + name as new parent + // id + if (this.get("parentIdPrefix")) { + var newParentId = this.get("parentIdPrefix") + this.get("id"); + this.set("parentId", newParentId); + console.log(newParentId); + } + this.initChart(); this.draw(); }, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/gpu-donut-chart.js hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/gpu-donut-chart.js new file mode 100644 index 00000000000..fa5ca8ac1dc --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/gpu-donut-chart.js @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import DonutChart from 'yarn-ui/components/donut-chart'; +import ColorUtils from 'yarn-ui/utils/color-utils'; + +export default DonutChart.extend({ + draw: function() { + // Construct data + var data = []; + if (this.get("gpu-render-type") === "gpu-memory") { + data.push({ + label: "Used", + value: parseFloat(this.get("gpuInfo").gpuMemoryUsage.usedMemoryMiB), + }); + data.push({ + label: "Available", + value: parseFloat(this.get("gpuInfo").gpuMemoryUsage.availMemoryMiB) + }); + } else if (this.get("gpu-render-type") === "gpu-utilization") { + var utilization = parseFloat(this.get("gpuInfo").gpuUtilizations.overallGpuUtilization); + data.push({ + label: "Utilized", + value: utilization, + }); + data.push({ + label: "Available", + value: 100 - utilization + }); + } + + var colorTargets = this.get("colorTargets"); + if (colorTargets) { + var colorTargetReverse = Boolean(this.get("colorTargetReverse")); + var targets = colorTargets.split(" "); + this.colors = ColorUtils.getColors(data.length, targets, colorTargetReverse); + } + + this.renderDonutChart(data, this.get("title"), this.get("showLabels"), + this.get("middleLabel"), this.get("middleValue"), this.get("suffix")); + }, + + didInsertElement: function() { + // ParentId includes minorNumber + var newParentId = this.get("parentId") + this.get("gpuInfo").minorNumber; + this.set("parentId", newParentId); + + this.initChart(); + this.draw(); + }, +}); \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js index d2937a0441f..29ad4bc2d93 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js @@ -22,3 +22,16 @@ export default { PARAM_SEPARATOR: '!', }; + +const BASE_UNIT = 1024 + +export const Type = 'type'; +export const Memory = 'memory'; +export const Resource = 'resource'; +export const Unit = 'unit'; +export const Entities = { + Type: 'type', + Memory:'memory', + Resource: 'resource', + Unit: 'unit' +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js index 3fae5961f87..f4bd5788433 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js @@ -60,7 +60,7 @@ export default Ember.Controller.extend({ getCellContent: function(row) { var node_id = row.get("id"), node_addr = row.get("nodeHTTPAddress"), - href = `#/yarn-node/${node_id}/${node_addr}`; + href = `#/yarn-node/${node_id}/${node_addr}/info`; switch(row.get("nodeState")) { case "SHUTDOWN": case "LOST": diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js index dcc0c2997d8..d9a5eefd769 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js @@ -43,6 +43,8 @@ export default DS.Model.extend({ decommissionedNodes: DS.attr('number'), rebootedNodes: DS.attr('number'), activeNodes: DS.attr('number'), + totalUsedResourcesAcrossPartition: DS.attr('object'), + totalClusterResourcesAcrossPartition: DS.attr('object'), getFinishedAppsDataForDonutChart: function() { var arr = []; @@ -135,4 +137,71 @@ export default DS.Model.extend({ return arr; }.property("allocatedVirtualCores", "reservedVirtualCores", "availableVirtualCores"), + + getResourceTypes: function() { + var types = []; + if (this.get("totalClusterResourcesAcrossPartition")) { + + console.log(types); + } + }.property("totalClusterResourcesAcrossPartition"), + + /* + * Returned format + * [ + * { + * name: + * unit: + * [ + * { + * label: