diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java index 2ec39762b8b..0fb6ab3f349 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java @@ -156,7 +156,7 @@ public void middleStep() throws Exception { float cpuUsed = node.getTotalCapability().getVirtualCores() * resourceUtilizationRatio; ResourceUtilization resourceUtilization = ResourceUtilization.newInstance( - pMemUsed, pMemUsed, cpuUsed); + pMemUsed, pMemUsed, cpuUsed, null); ns.setContainersUtilization(resourceUtilization); ns.setNodeUtilization(resourceUtilization); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java index f6c5a69eade..87562adca3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java @@ -22,6 +22,9 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.util.Records; +import java.util.HashMap; +import java.util.Map; + /** *

* ResourceUtilization models the utilization of a set of computer @@ -33,14 +36,19 @@ public abstract class ResourceUtilization implements Comparable { + private Map customResources + = new HashMap<>(); + @Public @Unstable - public static ResourceUtilization newInstance(int pmem, int vmem, float cpu) { + public static ResourceUtilization newInstance(int pmem, int vmem, + float cpu, Map customResources) { ResourceUtilization utilization = Records.newRecord(ResourceUtilization.class); utilization.setPhysicalMemory(pmem); utilization.setVirtualMemory(vmem); utilization.setCPU(cpu); + utilization.setCustomResources(customResources); return utilization; } @@ -49,7 +57,9 @@ public static ResourceUtilization newInstance(int pmem, int vmem, float cpu) { public static ResourceUtilization newInstance( ResourceUtilization resourceUtil) { return newInstance(resourceUtil.getPhysicalMemory(), - resourceUtil.getVirtualMemory(), resourceUtil.getCPU()); + resourceUtil.getVirtualMemory(), + resourceUtil.getCPU(), + resourceUtil.getCustomResources()); } /** @@ -106,6 +116,50 @@ public static ResourceUtilization newInstance( @Unstable public abstract void setCPU(float cpu); + /** + * Get custom resource utilization (The amount of custom resource used). + * + * @param resourceName resourceName of custom resource + * @return resourceName utilization + */ + @Public + @Unstable + public float getCustomResource(String resourceName) { + if (customResources != null && resourceName != null) { + return customResources.get(resourceName); + } + return 0f; + } + + @Public + @Unstable + public Map getCustomResources() { + return customResources; + } + + @Public + @Unstable + public void setCustomResources(Map customResources) { + if (customResources != null) { + this.customResources = customResources; + } + } + + /** + * Set custom resource utilization (The amount of custom resource used). + * @param resourceName resourceName + * @param utilization utilization of custom resource + * + */ + @Public + @Unstable + public void setCustomResource(String resourceName, float utilization) { + if (customResources != null && + resourceName != null && !resourceName.isEmpty()) { + customResources.put(resourceName, utilization); + } + } + @Override public int hashCode() { final int prime = 263167; @@ -113,6 +167,12 @@ public int hashCode() { result = prime * result + getVirtualMemory(); result = prime * result + getPhysicalMemory(); result = 31 * result + Float.valueOf(getCPU()).hashCode(); + if (customResources != null && !customResources.isEmpty()) { + for (Map.Entry entry : customResources.entrySet()) { + result = 31 * result + Float. + valueOf(customResources.get(entry.getValue())).hashCode(); + } + } return result; } @@ -130,7 +190,8 @@ public boolean equals(Object obj) { ResourceUtilization other = (ResourceUtilization) obj; if (getVirtualMemory() != other.getVirtualMemory() || getPhysicalMemory() != other.getPhysicalMemory() - || getCPU() != other.getCPU()) { + || getCPU() != other.getCPU() + || !customResources.equals(other.customResources) ) { return false; } return true; @@ -138,8 +199,17 @@ public boolean equals(Object obj) { @Override public String toString() { - return ""; + String utilizationString = + " entry : customResources.entrySet()) { + utilizationString += ", " + entry.getKey() + ":" + entry.getValue(); + } + } + + utilizationString += ">"; + return utilizationString; } /** @@ -147,13 +217,18 @@ public String toString() { * @param pmem Physical memory used to add. * @param vmem Virtual memory used to add. * @param cpu CPU utilization to add. + * @param resourceName of custom resource to add. + * @param utilization of custom resource to add. */ @Public @Unstable - public void addTo(int pmem, int vmem, float cpu) { + public void addTo(int pmem, int vmem, float cpu, + String resourceName, float utilization) { this.setPhysicalMemory(this.getPhysicalMemory() + pmem); this.setVirtualMemory(this.getVirtualMemory() + vmem); this.setCPU(this.getCPU() + cpu); + this.setCustomResource(resourceName, + getCustomResource(resourceName) + utilization); } /** @@ -161,12 +236,17 @@ public void addTo(int pmem, int vmem, float cpu) { * @param pmem Physical memory to be subtracted. * @param vmem Virtual memory to be subtracted. * @param cpu CPU utilization to be subtracted. + * @param resourceName of custom resource to be subtracted. + * @param utilization of custom resource to be subtracted. */ @Public @Unstable - public void subtractFrom(int pmem, int vmem, float cpu) { + public void subtractFrom(int pmem, int vmem, float cpu, + String resourceName, float utilization) { this.setPhysicalMemory(this.getPhysicalMemory() - pmem); this.setVirtualMemory(this.getVirtualMemory() - vmem); this.setCPU(this.getCPU() - cpu); + this.setCustomResource(resourceName, + getCustomResource(resourceName) - utilization); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 0b360dfca59..558e724f8ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -83,6 +83,7 @@ message ResourceUtilizationProto { optional int32 pmem = 1; optional int32 vmem = 2; optional float cpu = 3; + repeated StringFloatMapProto customResources = 4; } message ResourceOptionProto { @@ -243,6 +244,11 @@ message StringLongMapProto { required int64 value = 2; } +message StringFloatMapProto { + required string key = 1; + required float value = 2; +} + message ApplicationResourceUsageReportProto { optional int32 num_used_containers = 1; optional int32 num_reserved_containers = 2; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 1ed7dab2b86..29a94c32b63 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -2081,9 +2081,9 @@ private void verifyUsageInfo(YarnCLI cli) throws Exception { .newRecord(Resource.class), 0, "", 0, nodeLabels, null, null); if (!emptyResourceUtilization) { ResourceUtilization containersUtilization = ResourceUtilization - .newInstance(1024, 2048, 4); + .newInstance(1024, 2048, 4, null); ResourceUtilization nodeUtilization = ResourceUtilization.newInstance( - 2048, 4096, 8); + 2048, 4096, 8, null); nodeReport.setAggregatedContainersUtilization(containersUtilization); nodeReport.setNodeUtilization(nodeUtilization); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java index cdeb417243e..a325c3e3056 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -587,6 +587,21 @@ public static ResourceTypes convertFromProtoFormat(ResourceTypesProto e) { return ret; } + public static List convertMapToStringFloatMapProtoList( + Map map) { + List ret = new ArrayList<>(); + if (map != null) { + for (Map.Entry entry : map.entrySet()) { + YarnProtos.StringFloatMapProto.Builder tmp = + YarnProtos.StringFloatMapProto.newBuilder(); + tmp.setKey(entry.getKey()); + tmp.setValue(entry.getValue()); + ret.add(tmp.build()); + } + } + return ret; + } + public static Map convertStringStringMapProtoListToMap( List pList) { Map ret = new HashMap<>(); @@ -600,6 +615,19 @@ public static ResourceTypes convertFromProtoFormat(ResourceTypesProto e) { return ret; } + public static Map convertStringFloatMapProtoListToMap( + List pList) { + Map ret = new HashMap<>(); + if (pList != null) { + for (YarnProtos.StringFloatMapProto p : pList) { + if (p.hasKey()) { + ret.put(p.getKey(), p.getValue()); + } + } + } + return ret; + } + public static List convertToProtoFormat( Map stringMap) { List pList = new ArrayList<>(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java index e37adbe63f9..f5f2caa5e70 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java @@ -24,6 +24,9 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceUtilizationProtoOrBuilder; import org.apache.hadoop.yarn.api.records.ResourceUtilization; +import java.util.HashMap; +import java.util.Map; + @Private @Unstable public class ResourceUtilizationPBImpl extends ResourceUtilization { @@ -90,6 +93,23 @@ public void setCPU(float cpu) { builder.setCpu(cpu); } + @Override + public Map getCustomResources() { + ResourceUtilizationProtoOrBuilder p = viaProto ? proto : builder; + return ProtoUtils. + convertStringFloatMapProtoListToMap(p. + getCustomResourcesList()); + } + + @Override + public void setCustomResources(Map customResources) { + if (customResources != null) { + maybeInitBuilder(); + builder.addAllCustomResources(ProtoUtils. + convertMapToStringFloatMapProtoList(customResources)); + } + } + @Override public int compareTo(ResourceUtilization other) { int diff = this.getPhysicalMemory() - other.getPhysicalMemory(); @@ -97,6 +117,11 @@ public int compareTo(ResourceUtilization other) { diff = this.getVirtualMemory() - other.getVirtualMemory(); if (diff == 0) { diff = Float.compare(this.getCPU(), other.getCPU()); + if (diff == 0) { + diff = this.getCustomResources().size() - + other.getCustomResources().size(); + // todo consider custom resource priority? + } } } return diff; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java index 5934846e2f3..174ebf0c0f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java @@ -25,11 +25,15 @@ @Test public void testResourceUtilization() { - ResourceUtilization u1 = ResourceUtilization.newInstance(10, 20, 0.5f); + ResourceUtilization u1 = ResourceUtilization.newInstance(10, 20, + 0.5f, null); ResourceUtilization u2 = ResourceUtilization.newInstance(u1); - ResourceUtilization u3 = ResourceUtilization.newInstance(10, 20, 0.5f); - ResourceUtilization u4 = ResourceUtilization.newInstance(20, 20, 0.5f); - ResourceUtilization u5 = ResourceUtilization.newInstance(30, 40, 0.8f); + ResourceUtilization u3 = ResourceUtilization.newInstance(10, 20, + 0.5f, null); + ResourceUtilization u4 = ResourceUtilization.newInstance(20, 20, + 0.5f, null); + ResourceUtilization u5 = ResourceUtilization.newInstance(30, 40, + 0.8f, null); Assert.assertEquals(u1, u2); Assert.assertEquals(u1, u3); @@ -50,14 +54,14 @@ public void testResourceUtilization() { Assert.assertEquals("", u1.toString()); - u1.addTo(10, 0, 0.0f); + u1.addTo(10, 0, 0.0f, null, 0f); Assert.assertNotEquals(u1, u2); Assert.assertEquals(u1, u4); - u1.addTo(10, 20, 0.3f); + u1.addTo(10, 20, 0.3f, null, 0f); Assert.assertEquals(u1, u5); - u1.subtractFrom(10, 20, 0.3f); + u1.subtractFrom(10, 20, 0.3f, null, 0f); Assert.assertEquals(u1, u4); - u1.subtractFrom(10, 0, 0.0f); + u1.subtractFrom(10, 0, 0.0f, null, 0f); Assert.assertEquals(u1, u3); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java index 7577b558997..1dacf0b74f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java @@ -30,6 +30,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; +import java.util.Map; + /** * Implementation of the node resource monitor. It periodically tracks the * resource utilization of the node and reports it to the NM. @@ -54,8 +57,11 @@ private GpuNodeResourceUpdateHandler gpuNodeResourceUpdateHandler; /** Current resource utilization of the node. */ + + private Map customResources = new HashMap<>(); + private ResourceUtilization nodeUtilization = - ResourceUtilization.newInstance(0, 0, 0f); + ResourceUtilization.newInstance(0, 0, 0f, customResources); private Context nmContext; /** @@ -165,22 +171,31 @@ public void run() { resourceCalculatorPlugin.getVirtualMemorySize() - resourceCalculatorPlugin.getAvailableVirtualMemorySize(); float vcores = resourceCalculatorPlugin.getNumVCoresUsed(); - nodeUtilization = - ResourceUtilization.newInstance( - (int) (pmem >> 20), // B -> MB - (int) (vmem >> 20), // B -> MB - vcores); // Used Virtual Cores float nodeGpuUtilization = 0F; + float nodeGpus = 0F; try { if (gpuNodeResourceUpdateHandler != null) { nodeGpuUtilization = gpuNodeResourceUpdateHandler.getNodeGpuUtilization(); + nodeGpus = + gpuNodeResourceUpdateHandler.getNodePhysGpus(); } } catch (Exception e) { LOG.error("Get Node GPU Utilization error: " + e); } + nodeUtilization = + ResourceUtilization.newInstance( + (int) (pmem >> 20), // B -> MB + (int) (vmem >> 20), // B -> MB + vcores, // Used Virtual Cores + customResources); // Used GPUs + + nodeUtilization. + setCustomResource(ResourceInformation.GPU_URI, nodeGpus); + + // Publish the node utilization metrics to node manager // metrics system. NodeManagerMetrics nmMetrics = nmContext.getNodeManagerMetrics(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java index 002035bcdc9..7597ab22598 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java @@ -46,7 +46,7 @@ static void increaseResourceUtilization( float vCores = (float) resource.getVirtualCores(); int vmem = (int) (resource.getMemorySize() * containersMonitor.getVmemRatio()); - resourceUtil.addTo((int)resource.getMemorySize(), vmem, vCores); + resourceUtil.addTo((int)resource.getMemorySize(), vmem, vCores, null, 0f); } /** @@ -62,7 +62,8 @@ static void decreaseResourceUtilization( float vCores = (float) resource.getVirtualCores(); int vmem = (int) (resource.getMemorySize() * containersMonitor.getVmemRatio()); - resourceUtil.subtractFrom((int)resource.getMemorySize(), vmem, vCores); + resourceUtil.subtractFrom((int)resource.getMemorySize(), + vmem, vCores, null, 0f); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index a83ae3ad036..2cfe370c087 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -133,7 +133,8 @@ public ContainersMonitorImpl(ContainerExecutor exec, this.logMonitorThread = new LogMonitorThread(); - this.containersUtilization = ResourceUtilization.newInstance(0, 0, 0.0f); + this.containersUtilization = ResourceUtilization.newInstance(0, 0, + 0.0f, null); } @Override @@ -511,7 +512,7 @@ public void run() { // Temporary structure to calculate the total resource utilization of // the containers ResourceUtilization trackedContainersUtilization = - ResourceUtilization.newInstance(0, 0, 0.0f); + ResourceUtilization.newInstance(0, 0, 0.0f, null); // Now do the monitoring for the trackingContainers // Check memory usage and kill any overflowing containers @@ -704,7 +705,7 @@ private void recordUsage(ContainerId containerId, String pId, trackedContainersUtilization.addTo( (int) (currentPmemUsage >> 20), (int) (currentVmemUsage >> 20), - milliVcoresUsed / 1000.0f); + milliVcoresUsed / 1000.0f, null, 0f); // Add usage to container metrics if (containerMetricsEnabled) { @@ -1049,7 +1050,7 @@ public void subtractNodeResourcesFromResourceUtilization( ResourceUtilization resourceUtil) { resourceUtil.subtractFrom((int) (getPmemAllocatedForContainers() >> 20), (int) (getVmemAllocatedForContainers() >> 20), - getVCoresAllocatedForContainers()); + getVCoresAllocatedForContainers(), null, 0f); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java index af81709566d..722b864b124 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java @@ -86,12 +86,19 @@ public float getNodeGpuUtilization() throws Exception{ if (gpuList != null && gpuList.size() != 0) { - totalGpuUtilization = gpuList - .stream() - .map(g -> g.getGpuUtilizations().getOverallGpuUtilization()) - .collect(Collectors.summingDouble(Float::floatValue)) - .floatValue() / gpuList.size(); + totalGpuUtilization = getNodePhysGpus() / gpuList.size(); } return totalGpuUtilization; } + + public float getNodePhysGpus() throws Exception{ + List gpuList = + gpuDiscoverer.getGpuDeviceInformation().getGpus(); + Float totalGpuUtilization = gpuList + .stream() + .map(g -> g.getGpuUtilizations().getOverallGpuUtilization()) + .collect(Collectors.summingDouble(Float::floatValue)) + .floatValue(); + return totalGpuUtilization; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java index 100676d27dd..4987991e1f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java @@ -38,7 +38,8 @@ private ContainerScheduler scheduler; AllocationBasedResourceUtilizationTracker(ContainerScheduler scheduler) { - this.containersAllocation = ResourceUtilization.newInstance(0, 0, 0.0f); + this.containersAllocation = + ResourceUtilization.newInstance(0, 0, 0.0f, null); this.scheduler = scheduler; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java index c67ae86f95e..c4bfd063e6b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java @@ -529,7 +529,7 @@ public void testContainerSchedulerRecovery() throws Exception { assertNotNull(app); ResourceUtilization utilization = - ResourceUtilization.newInstance(1024, 2048, 1.0F); + ResourceUtilization.newInstance(1024, 2048, 1.0F, null); assertThat(cm.getContainerScheduler().getNumRunningContainers()). isEqualTo(1); assertEquals(utilization, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java index cc8e180b6c6..c6b75500f4b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java @@ -303,7 +303,7 @@ public void testContainersCPUResourceForDefaultValue() throws Exception { assertEquals( "Resource utilization must be default with MonitorThread's first run", 0, containersMonitor.getContainersUtilization() - .compareTo(ResourceUtilization.newInstance(0, 0, 0.0f))); + .compareTo(ResourceUtilization.newInstance(0, 0, 0.0f, null))); // Verify the container utilization value. Since atleast one round is done, // we can expect a non-zero value for container utilization as @@ -318,7 +318,7 @@ public static void waitForContainerResourceUtilizationChange( throws InterruptedException { int timeWaiting = 0; while (0 == containersMonitor.getContainersUtilization() - .compareTo(ResourceUtilization.newInstance(0, 0, 0.0f))) { + .compareTo(ResourceUtilization.newInstance(0, 0, 0.0f, null))) { if (timeWaiting >= timeoutMsecs) { break; } @@ -331,7 +331,7 @@ public static void waitForContainerResourceUtilizationChange( assertTrue("Resource utilization is not changed from second run onwards", 0 != containersMonitor.getContainersUtilization() - .compareTo(ResourceUtilization.newInstance(0, 0, 0.0f))); + .compareTo(ResourceUtilization.newInstance(0, 0, 0.0f, null))); } private ContainersMonitorImpl createContainersMonitor( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java index b09a1f260e0..c7e3d098f70 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java @@ -53,7 +53,7 @@ private static final Resource CONTAINER_SIZE = Resource.newInstance(1024, 4); private static final ResourceUtilization ZERO = - ResourceUtilization.newInstance(0, 0, 0.0f); + ResourceUtilization.newInstance(0, 0, 0.0f, null); @Mock private NMContext context; @@ -206,7 +206,7 @@ private void setupContainerMonitor() { assertEquals(0, spy.getNumQueuedOpportunisticContainers()); assertEquals(1, spy.getNumRunningContainers()); assertEquals( - ResourceUtilization.newInstance(1024, 1024, 4.0f), + ResourceUtilization.newInstance(1024, 1024, 4.0f, null), spy.getCurrentUtilization()); } @@ -227,7 +227,7 @@ private void setupContainerMonitor() { assertEquals(0, spy.getNumQueuedOpportunisticContainers()); assertEquals(1, spy.getNumRunningContainers()); assertEquals( - ResourceUtilization.newInstance(1024, 1024, 4.0f), + ResourceUtilization.newInstance(1024, 1024, 4.0f, null), spy.getCurrentUtilization()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java index 468f6217ecd..0651047e167 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java @@ -69,9 +69,9 @@ private RMContainer reservedContainer; private volatile int numContainers; private volatile ResourceUtilization containersUtilization = - ResourceUtilization.newInstance(0, 0, 0f); + ResourceUtilization.newInstance(0, 0, 0f, null); private volatile ResourceUtilization nodeUtilization = - ResourceUtilization.newInstance(0, 0, 0f); + ResourceUtilization.newInstance(0, 0, 0f, null); /** Time stamp for overcommitted resources to time out. */ private long overcommitTimeout = -1; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java index dad27839cf6..9f37bcfd3bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java @@ -1209,7 +1209,7 @@ public void testCalculateHeartBeatInterval() { float slowdown = 1.0F; metrics.incrUtilizedVirtualCores(vcoreUnit * 5); // 50 % cluster util ResourceUtilization nodeUtil = ResourceUtilization.newInstance( - 1024, vcoreUnit, 0.0F * vcoreUnit); // 0% rmNode util + 1024, vcoreUnit, 0.0F * vcoreUnit, null); // 0% rmNode util calcIntervalTest(rmNode, nodeUtil, hbDefault, hbMin, hbMax, speedup, slowdown, vcoreUnit * 0.0F, hbMin); // 0% diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java index c67c49a3610..9fe7a82339c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java @@ -739,9 +739,9 @@ public void testNodesResourceUtilization() throws JSONException, Exception { NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(true, "test health report", System.currentTimeMillis()); ResourceUtilization nodeResource = ResourceUtilization.newInstance(4096, 0, - (float) 10.5); + (float) 10.5, null); ResourceUtilization containerResource = ResourceUtilization.newInstance( - 2048, 0, (float) 5.05); + 2048, 0, (float) 5.05, null); NodeStatus nodeStatus = NodeStatus.newInstance(nodeId1, 0, new ArrayList(), null, nodeHealth, containerResource, @@ -919,14 +919,15 @@ public void verifyNodeInfoGeneric(RMNode node, String state, String rack, node.getNodeManagerVersion(), version); if (node.getNodeUtilization() != null) { ResourceUtilization nodeResource = ResourceUtilization.newInstance( - nodePhysicalMemoryMB, nodeVirtualMemoryMB, (float) nodeCPUUsage); + nodePhysicalMemoryMB, nodeVirtualMemoryMB, + (float) nodeCPUUsage, null); assertEquals("nodeResourceUtilization doesn't match", node.getNodeUtilization(), nodeResource); } if (node.getAggregatedContainersUtilization() != null) { ResourceUtilization containerResource = ResourceUtilization.newInstance( containersPhysicalMemoryMB, containersVirtualMemoryMB, - (float) containersCPUUsage); + (float) containersCPUUsage, null); assertEquals("containerResourceUtilization doesn't match", node.getAggregatedContainersUtilization(), containerResource); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java index d1ccc9a6fec..b035200d0d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java @@ -165,9 +165,9 @@ private NodeStatus createNodeStatus( // Fake node status with fake utilization ResourceUtilization containersUtilization = ResourceUtilization.newInstance(containerPMem, containerVMem, - containerCPU); + containerCPU, null); ResourceUtilization nodeUtilization = - ResourceUtilization.newInstance(nodePMem, nodeVMem, nodeCPU); + ResourceUtilization.newInstance(nodePMem, nodeVMem, nodeCPU, null); NodeStatus status = NodeStatus.newInstance( nodeId, responseId,