diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java index cf4e020..311462b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java @@ -61,8 +61,8 @@ private static String getAppsTableColumnDefs( // Update following line if any column added in RM page before column 11 sb.append("{'sType':'num-ignore-str', ") .append("'aTargets': [12, 13, 14, 15, 16] },\n"); - // set progress column index to 19 - progressIndex = "[19]"; + // set progress column index to 21 + progressIndex = "[21]"; } else if (isFairSchedulerPage) { // Update following line if any column added in scheduler page before column 11 sb.append("{'sType':'num-ignore-str', ") @@ -112,4 +112,4 @@ public static String resourceRequestsTableInit() { .toString(); } -} \ No newline at end of file +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index d053f33..8afc61a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -28,9 +28,12 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.StringHelper; @Public @@ -63,8 +66,10 @@ protected int priority; private long allocatedCpuVcores; private long allocatedMemoryMB; + private long allocatedGpus; private long reservedCpuVcores; private long reservedMemoryMB; + private long reservedGpus; protected boolean unmanagedApplication; private String appNodeLabelExpression; private String amNodeLabelExpression; @@ -112,6 +117,18 @@ public AppInfo(ApplicationReport app) { .getReservedResources().getVirtualCores(); reservedMemoryMB = app.getApplicationResourceUsageReport() .getReservedResources().getMemorySize(); + ApplicationResourceUsageReport usageReport = + app.getApplicationResourceUsageReport(); + Integer gpuIndex = ResourceUtils.getResourceTypeIndex() + .get(ResourceInformation.GPU_URI); + allocatedGpus = -1; + reservedGpus = -1; + if (gpuIndex != null) { + allocatedGpus = usageReport.getUsedResources() + .getResourceValue(ResourceInformation.GPU_URI); + reservedGpus = usageReport.getReservedResources() + .getResourceValue(ResourceInformation.GPU_URI); + } } aggregateResourceAllocation = StringHelper.getResourceSecondsString( app.getApplicationResourceUsageReport().getResourceSecondsMap()); @@ -176,6 +193,10 @@ public long getAllocatedMemoryMB() { return allocatedMemoryMB; } + public long getAllocatedGpus() { + return allocatedGpus; + } + public long getReservedCpuVcores() { return reservedCpuVcores; } @@ -184,6 +205,10 @@ public long getReservedMemoryMB() { return reservedMemoryMB; } + public long getReservedGpus() { + return reservedGpus; + } + public float getProgress() { return progress; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java index 806b636..e56b677 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java @@ -70,12 +70,9 @@ protected void render(Block html) { th().$class("ui-state-default").__("Apps Running").__(). th().$class("ui-state-default").__("Apps Completed").__(). th().$class("ui-state-default").__("Containers Running").__(). - th().$class("ui-state-default").__("Memory Used").__(). - th().$class("ui-state-default").__("Memory Total").__(). - th().$class("ui-state-default").__("Memory Reserved").__(). - th().$class("ui-state-default").__("VCores Used").__(). - th().$class("ui-state-default").__("VCores Total").__(). - th().$class("ui-state-default").__("VCores Reserved").__(). + th().$class("ui-state-default").__("Used Resources").__(). + th().$class("ui-state-default").__("Total Resources").__(). + th().$class("ui-state-default").__("Reserved Resources").__(). __(). __(). tbody().$class("ui-widget-content"). @@ -90,12 +87,15 @@ protected void render(Block html) { ) ). td(String.valueOf(clusterMetrics.getContainersAllocated())). - td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)). - td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)). - td(StringUtils.byteDesc(clusterMetrics.getReservedMB() * BYTES_IN_MB)). - td(String.valueOf(clusterMetrics.getAllocatedVirtualCores())). - td(String.valueOf(clusterMetrics.getTotalVirtualCores())). - td(String.valueOf(clusterMetrics.getReservedVirtualCores())). + td(clusterMetrics.getTotalUsedResourcesAcrossPartition() == null ? "N/A" + : clusterMetrics.getTotalUsedResourcesAcrossPartition().getResource() + .toString()). + td(clusterMetrics.getTotalClusterResourcesAcrossPartition() == null ? + "N/A" : clusterMetrics.getTotalClusterResourcesAcrossPartition() + .getResource().toString()). + td(clusterMetrics.getTotalReservedResourcesAcrossPartition() == null ? + "N/A" : clusterMetrics.getTotalReservedResourcesAcrossPartition() + .getResource().toString()). __(). __().__(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java index 8e79740..199bdd5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java @@ -22,6 +22,7 @@ import org.apache.commons.text.StringEscapeUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; @@ -30,6 +31,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE; @@ -86,7 +88,9 @@ protected void render(Block html) { .th(".mem", "Mem Used") .th(".mem", "Mem Avail") .th(".vcores", "VCores Used") - .th(".vcores", "VCores Avail"); + .th(".vcores", "VCores Avail") + .th(".vcores", "GPUs Used") + .th(".vcores", "GPUs Avail"); } else { trbody.th(".containers", "Running Containers (G)") .th(".allocationTags", "Allocation Tags") @@ -94,6 +98,8 @@ protected void render(Block html) { .th(".mem", "Mem Avail (G)") .th(".vcores", "VCores Used (G)") .th(".vcores", "VCores Avail (G)") + .th(".vcores", "GPUs Used (G)") + .th(".vcores", "GPUs Avail (G)") .th(".containers", "Running Containers (O)") .th(".mem", "Mem Used (O)") .th(".vcores", "VCores Used (O)") @@ -165,6 +171,16 @@ protected void render(Block html) { nodeTableData.append("\",\"").append(httpAddress).append("\",").append("\""); } + Integer gpuIndex = ResourceUtils.getResourceTypeIndex() + .get(ResourceInformation.GPU_URI); + long usedGPUs = 0; + long availableGPUs = 0; + if (gpuIndex != null) { + usedGPUs = info.getUsedResource().getResource() + .getResourceValue(ResourceInformation.GPU_URI); + availableGPUs = info.getAvailableResource().getResource() + .getResourceValue(ResourceInformation.GPU_URI); + } nodeTableData.append("
") .append(Times.format(info.getLastHealthUpdate())).append("\",\"") @@ -179,6 +195,10 @@ protected void render(Block html) { .append("\",\"").append(String.valueOf(info.getUsedVirtualCores())) .append("\",\"") .append(String.valueOf(info.getAvailableVirtualCores())) + .append("\",\"") + .append(String.valueOf(usedGPUs)) + .append("\",\"") + .append(String.valueOf(availableGPUs)) .append("\",\""); // If opportunistic containers are enabled, add extra fields. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java index e8da0cf..09dec9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java @@ -69,8 +69,10 @@ new ColumnHeader(".runningcontainer", "Running Containers"), new ColumnHeader(".allocatedCpu", "Allocated CPU VCores"), new ColumnHeader(".allocatedMemory", "Allocated Memory MB"), + new ColumnHeader(".allocatedGpu", "Allocated GPUs"), new ColumnHeader(".reservedCpu", "Reserved CPU VCores"), new ColumnHeader(".reservedMemory", "Reserved Memory MB"), + new ColumnHeader(".reservedGpu", "Reserved GPUs"), new ColumnHeader(".queuePercentage", "% of Queue"), new ColumnHeader(".clusterPercentage", "% of Cluster"), new ColumnHeader(".progress", "Progress"), @@ -119,6 +121,7 @@ protected void renderData(Block html) { String blacklistedNodesCount = "N/A"; RMApp rmApp = rm.getRMContext().getRMApps() .get(appAttemptId.getApplicationId()); + boolean isAppInCompletedState = false; if (rmApp != null) { RMAppAttempt appAttempt = rmApp.getRMAppAttempt(appAttemptId); Set nodes = @@ -126,6 +129,7 @@ protected void renderData(Block html) { if (nodes != null) { blacklistedNodesCount = String.valueOf(nodes.size()); } + isAppInCompletedState = rmApp.isAppInCompletedStates(); } String percent = StringUtils.format("%.1f", app.getProgress()); appsTableData @@ -171,12 +175,18 @@ protected void renderData(Block html) { .append(app.getAllocatedMemoryMB() == -1 ? "N/A" : String.valueOf(app.getAllocatedMemoryMB())) .append("\",\"") + .append((isAppInCompletedState && app.getAllocatedGpus() <= 0) + ? "N/A" : String.valueOf(app.getAllocatedGpus())) + .append("\",\"") .append(app.getReservedCpuVcores() == -1 ? "N/A" : String .valueOf(app.getReservedCpuVcores())) .append("\",\"") .append(app.getReservedMemoryMB() == -1 ? "N/A" : String.valueOf(app.getReservedMemoryMB())) .append("\",\"") + .append((isAppInCompletedState && app.getReservedGpus() <= 0) + ? "N/A" : String.valueOf(app.getReservedGpus())) + .append("\",\"") .append(queuePercent) .append("\",\"") .append(clusterPercent) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java index d6e4828..1ed9f00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java @@ -69,6 +69,9 @@ // Total registered resources of the cluster, including all partitions private ResourceInfo totalClusterResourcesAcrossPartition; + // Total registered resources of the cluster, including all partitions + private ResourceInfo totalReservedResourcesAcrossPartition; + public ClusterMetricsInfo() { } // JAXB needs this @@ -115,6 +118,8 @@ public ClusterMetricsInfo(final ResourceScheduler rs) { cs.getRootQueue().getQueueResourceUsage().getAllUsed()); totalClusterResourcesAcrossPartition = new ResourceInfo( cs.getClusterResource()); + totalReservedResourcesAcrossPartition = new ResourceInfo( + cs.getClusterResourceUsage().getReserved()); } } else { this.totalMB = availableMB + allocatedMB; @@ -346,4 +351,8 @@ public ResourceInfo getTotalUsedResourcesAcrossPartition() { public ResourceInfo getTotalClusterResourcesAcrossPartition() { return totalClusterResourcesAcrossPartition; } + + public ResourceInfo getTotalReservedResourcesAcrossPartition() { + return totalReservedResourcesAcrossPartition; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java index 26e8c2a..1f6a8c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java @@ -48,8 +48,8 @@ // Number of Actual Table Headers for NodesPage.NodesBlock might change in // future. In that case this value should be adjusted to the new value. - private final int numberOfThInMetricsTable = 23; - private final int numberOfActualTableHeaders = 14; + private final int numberOfThInMetricsTable = 20; + private final int numberOfActualTableHeaders = 16; private final int numberOfThForOpportunisticContainers = 4; private Injector injector;