diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index 27469a8..b52a9ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -30,8 +30,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; @Public @Evolving @@ -63,8 +65,10 @@ protected int priority; private long allocatedCpuVcores; private long allocatedMemoryMB; + private long allocatedGpus; private long reservedCpuVcores; private long reservedMemoryMB; + private long reservedGpus; protected boolean unmanagedApplication; private String appNodeLabelExpression; private String amNodeLabelExpression; @@ -110,6 +114,16 @@ public AppInfo(ApplicationReport app) { reservedCpuVcores = usageReport.getReservedResources(). getVirtualCores(); reservedMemoryMB = usageReport.getReservedResources().getMemorySize(); + Integer gpuIndex = ResourceUtils.getResourceTypeIndex() + .get(ResourceInformation.GPU_URI); + allocatedGpus = -1; + reservedGpus = -1; + if (gpuIndex != null) { + allocatedGpus = usageReport.getUsedResources() + .getResourceValue(ResourceInformation.GPU_URI); + reservedGpus = usageReport.getReservedResources() + .getResourceValue(ResourceInformation.GPU_URI); + } } aggregateResourceAllocation = usageReport.getMemorySeconds() + " MB-seconds, " + usageReport.getVcoreSeconds() @@ -175,6 +189,10 @@ public long getAllocatedMemoryMB() { return allocatedMemoryMB; } + public long getAllocatedGpus() { + return allocatedGpus; + } + public long getReservedCpuVcores() { return reservedCpuVcores; } @@ -183,6 +201,10 @@ public long getReservedMemoryMB() { return reservedMemoryMB; } + public long getReservedGpus() { + return reservedGpus; + } + public float getProgress() { return progress; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java index f51f0c5..da84d5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java @@ -57,6 +57,7 @@ protected void commonPreHead(HTML<_> html) { @Inject public NodeBlock(Context context, ResourceView resourceView) { +new Throwable("DBG-0:").printStackTrace(System.out); this.context = context; this.resourceView = resourceView; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java index aebac0c..a417587 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java @@ -70,12 +70,10 @@ protected void render(Block html) { th().$class("ui-state-default")._("Apps Running")._(). th().$class("ui-state-default")._("Apps Completed")._(). th().$class("ui-state-default")._("Containers Running")._(). - th().$class("ui-state-default")._("Memory Used")._(). - th().$class("ui-state-default")._("Memory Total")._(). - th().$class("ui-state-default")._("Memory Reserved")._(). th().$class("ui-state-default")._("VCores Used")._(). - th().$class("ui-state-default")._("VCores Total")._(). - th().$class("ui-state-default")._("VCores Reserved")._(). + th().$class("ui-state-default")._("Used Resources")._(). + th().$class("ui-state-default")._("Total Resources")._(). + th().$class("ui-state-default")._("Reserved Resources")._(). _(). _(). tbody().$class("ui-widget-content"). @@ -90,12 +88,13 @@ protected void render(Block html) { ) ). td(String.valueOf(clusterMetrics.getContainersAllocated())). - td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)). - td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)). - td(StringUtils.byteDesc(clusterMetrics.getReservedMB() * BYTES_IN_MB)). td(String.valueOf(clusterMetrics.getAllocatedVirtualCores())). - td(String.valueOf(clusterMetrics.getTotalVirtualCores())). - td(String.valueOf(clusterMetrics.getReservedVirtualCores())). + td(clusterMetrics.getTotalUsedResourcesAcrossPartition().getResource() + .toString()). + td(clusterMetrics.getTotalClusterResourcesAcrossPartition().getResource() + .toString()). + td(clusterMetrics.getTotalReservedResourcesAcrossPartition().getResource() + .toString()). _(). _()._(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java index e080f59..275cd1f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java @@ -22,6 +22,7 @@ import org.apache.commons.lang.StringEscapeUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; @@ -30,6 +31,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; @@ -85,7 +87,9 @@ protected void render(Block html) { .th(".mem", "Mem Used") .th(".mem", "Mem Avail") .th(".vcores", "VCores Used") - .th(".vcores", "VCores Avail"); + .th(".vcores", "VCores Avail") + .th(".vcores", "GPUs Used") + .th(".vcores", "GPUs Avail"); } else { trbody.th(".containers", "Running Containers (G)") .th(".mem", "Mem Used (G)") @@ -163,6 +167,16 @@ protected void render(Block html) { nodeTableData.append("\",\"").append(httpAddress).append("\",").append("\""); } + Integer gpuIndex = ResourceUtils.getResourceTypeIndex() + .get(ResourceInformation.GPU_URI); + long usedGPUs = 0; + long availableGPUs = 0; + if (gpuIndex != null) { + usedGPUs = info.getUsedResource().getResource() + .getResourceValue(ResourceInformation.GPU_URI); + availableGPUs = info.getAvailableResource().getResource() + .getResourceValue(ResourceInformation.GPU_URI); + } nodeTableData.append("
") .append(Times.format(info.getLastHealthUpdate())).append("\",\"") @@ -176,7 +190,12 @@ protected void render(Block html) { .append("\",\"").append(String.valueOf(info.getUsedVirtualCores())) .append("\",\"") .append(String.valueOf(info.getAvailableVirtualCores())) - .append("\",\""); + .append("\",\"") + .append(String.valueOf(usedGPUs)) + .append("\",\"") + .append(String.valueOf(availableGPUs)) + .append("\",\"") + ; // If opportunistic containers are enabled, add extra fields. if (this.opportunisticContainersEnabled) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java index e0baf10..5600c11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java @@ -38,7 +38,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.webapp.AppsBlock; import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.View; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; @@ -69,8 +68,10 @@ protected void renderData(Block html) { .th(".runningcontainer", "Running Containers") .th(".allocatedCpu", "Allocated CPU VCores") .th(".allocatedMemory", "Allocated Memory MB") + .th(".allocatedGpu", "Allocated GPUs") .th(".reservedCpu", "Reserved CPU VCores") .th(".reservedMemory", "Reserved Memory MB") + .th(".reservedGpu", "Reserved GPUs") .th(".queuePercentage", "% of Queue") .th(".clusterPercentage", "% of Cluster") .th(".progress", "Progress") @@ -152,12 +153,18 @@ protected void renderData(Block html) { .append(app.getAllocatedMemoryMB() == -1 ? "N/A" : String.valueOf(app.getAllocatedMemoryMB())) .append("\",\"") + .append(app.getAllocatedGpus() == -1 ? "N/A" : String + .valueOf(app.getAllocatedGpus())) + .append("\",\"") .append(app.getReservedCpuVcores() == -1 ? "N/A" : String .valueOf(app.getReservedCpuVcores())) .append("\",\"") .append(app.getReservedMemoryMB() == -1 ? "N/A" : String.valueOf(app.getReservedMemoryMB())) .append("\",\"") + .append(app.getReservedGpus() == -1 ? "N/A" : String + .valueOf(app.getReservedGpus())) + .append("\",\"") .append(queuePercent) .append("\",\"") .append(clusterPercent) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java index 84f70d9..2e7955c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java @@ -67,6 +67,9 @@ // Total registered resources of the cluster, including all partitions private ResourceInfo totalClusterResourcesAcrossPartition; + // Total registered resources of the cluster, including all partitions + private ResourceInfo totalReservedResourcesAcrossPartition; + public ClusterMetricsInfo() { } // JAXB needs this @@ -111,6 +114,8 @@ public ClusterMetricsInfo(final ResourceScheduler rs) { cs.getRootQueue().getQueueResourceUsage().getAllUsed()); totalClusterResourcesAcrossPartition = new ResourceInfo( cs.getClusterResource()); + totalReservedResourcesAcrossPartition = new ResourceInfo( + cs.getClusterResourceUsage().getReserved()); } } else { this.totalMB = availableMB + allocatedMB; @@ -334,4 +339,8 @@ public ResourceInfo getTotalUsedResourcesAcrossPartition() { public ResourceInfo getTotalClusterResourcesAcrossPartition() { return totalClusterResourcesAcrossPartition; } + + public ResourceInfo getTotalReservedResourcesAcrossPartition() { + return totalReservedResourcesAcrossPartition; + } }