diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java new file mode 100644 index 0000000..b8ad66e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +15 * See the License for the specific language governing permissions and +16 * limitations under the License. +17 */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * ResourceUtilization models the utilization of a set of computer + * resources in the cluster. + *

+ * + * @see Resource + */ +@Public +@Evolving +public abstract class ResourceUtilization implements + Comparable { + @Public + @Evolving + public static ResourceUtilization newInstance(int pmem, int vmem, float cpu) { + ResourceUtilization utilization = + Records.newRecord(ResourceUtilization.class); + utilization.setPhysicalMemory(pmem); + utilization.setVirtualMemory(vmem); + utilization.setCPUUsage(cpu); + return utilization; + } + + /** + * Get used virtual memory. + * + * @return virtual memory in MB + */ + @Public + @Evolving + public abstract int getVirtualMemory(); + + /** + * Set used virtual memory. + * + * @param vmem virtual memory in MB + */ + @Public + @Evolving + public abstract void setVirtualMemory(int vmem); + + /** + * Get physical memory. + * + * @return physical memory in MB + */ + @Public + @Evolving + public abstract int getPhysicalMemory(); + + /** + * Set physical memory. + * + * @param pmem physical memory in MB + */ + @Public + @Evolving + public abstract void setPhysicalMemory(int pmem); + + /** + * Get CPU utilization. + * + * @return CPU utilization normalized to 1 CPU + */ + @Public + @Evolving + public abstract float getCPUUsage(); + + /** + * Set CPU utilization. + * + * @param cpu CPU utilization normalized to 1 CPU + */ + @Public + @Evolving + public abstract void setCPUUsage(float cpu); + + @Override + public int hashCode() { + final int prime = 263167; + int result = 3571; + result = 939769357 + getVirtualMemory(); // prime * result = 939769357 + // initially + result = prime * result + getPhysicalMemory(); + result = 31 * result + Float.valueOf(getCPUUsage()).hashCode(); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof ResourceUtilization)) { + return false; + } + ResourceUtilization other = (ResourceUtilization) obj; + if (getVirtualMemory() != other.getVirtualMemory() + || getPhysicalMemory() != other.getPhysicalMemory() + || getCPUUsage() != other.getCPUUsage()) { + return false; + } + return true; + } + + @Override + public String toString() { + return ""; + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index f801409..47b731b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -58,6 +58,12 @@ message ResourceProto { optional int32 virtual_cores = 2; } +message ResourceUtilizationProto { + optional int32 pmem = 1; + optional int32 vmem = 2; + optional float cpu = 3; +} + message ResourceOptionProto { optional ResourceProto resource = 1; optional int32 over_commit_timeout = 2; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java new file mode 100644 index 0000000..a24df77 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ResourceUtilization; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceUtilizationProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceUtilizationProtoOrBuilder; + +@Private +@Unstable +public class ResourceUtilizationPBImpl extends ResourceUtilization { + ResourceUtilizationProto proto = ResourceUtilizationProto + .getDefaultInstance(); + ResourceUtilizationProto.Builder builder = null; + boolean viaProto = false; + + public ResourceUtilizationPBImpl() { + builder = ResourceUtilizationProto.newBuilder(); + } + + public ResourceUtilizationPBImpl(ResourceUtilizationProto proto) { + this.proto = proto; + viaProto = true; + } + + public ResourceUtilizationProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ResourceUtilizationProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public int getPhysicalMemory() { + ResourceUtilizationProtoOrBuilder p = viaProto ? proto : builder; + return (p.getPmem()); + } + + @Override + public void setPhysicalMemory(int pmem) { + maybeInitBuilder(); + builder.setPmem(pmem); + } + + @Override + public int getVirtualMemory() { + ResourceUtilizationProtoOrBuilder p = viaProto ? proto : builder; + return (p.getVmem()); + } + + @Override + public void setVirtualMemory(int vmem) { + maybeInitBuilder(); + builder.setPmem(vmem); + } + + @Override + public float getCPUUsage() { + ResourceUtilizationProtoOrBuilder p = viaProto ? proto : builder; + return p.getCpu(); + } + + @Override + public void setCPUUsage(float cpu) { + maybeInitBuilder(); + builder.setCpu(cpu); + } + + @Override + public int compareTo(ResourceUtilization other) { + int diff = this.getPhysicalMemory() - other.getPhysicalMemory(); + if (diff == 0) { + diff = this.getVirtualMemory() - other.getVirtualMemory(); + if (diff == 0) { + diff = Float.compare(this.getCPUUsage(), other.getCPUUsage()); + } + } + return diff; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtilizations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtilizations.java new file mode 100644 index 0000000..ef65be6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtilizations.java @@ -0,0 +1,55 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.util.resource; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ResourceUtilization; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"}) +@Unstable +public class ResourceUtilizations { + public static ResourceUtilization createResource(int pmem, int vmem, float cpu) { + ResourceUtilization utilization = + Records.newRecord(ResourceUtilization.class); + utilization.setPhysicalMemory(pmem); + utilization.setVirtualMemory(vmem); + utilization.setCPUUsage(cpu); + return utilization; + } + + public static ResourceUtilization clone(ResourceUtilization res) { + return createResource(res.getPhysicalMemory(), res.getVirtualMemory(), + res.getCPUUsage()); + } + + public static ResourceUtilization addTo(ResourceUtilization lhs, + ResourceUtilization rhs) { + lhs.setPhysicalMemory(lhs.getPhysicalMemory() + rhs.getPhysicalMemory()); + lhs.setVirtualMemory(lhs.getVirtualMemory() + rhs.getVirtualMemory()); + lhs.setCPUUsage(lhs.getCPUUsage() + rhs.getCPUUsage()); + return lhs; + } + + public static ResourceUtilization add(ResourceUtilization lhs, + ResourceUtilization rhs) { + return addTo(clone(lhs), rhs); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 8dacd3b..9c91041 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -140,6 +140,7 @@ import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.StrictPreemptionContract; import org.apache.hadoop.yarn.api.records.Token; @@ -175,6 +176,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ResourceOptionPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ResourceRequestPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ResourceUtilizationPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.StrictPreemptionContractPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl; @@ -209,6 +211,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceOptionProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceUtilizationProto; import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto; import org.apache.hadoop.yarn.proto.YarnProtos.StrictPreemptionContractProto; import org.apache.hadoop.yarn.proto.YarnProtos.URLProto; @@ -453,6 +456,7 @@ public static void setup() throws Exception { generateByNewInstance(Resource.class); generateByNewInstance(ResourceBlacklistRequest.class); generateByNewInstance(ResourceOption.class); + generateByNewInstance(ResourceUtilization.class); generateByNewInstance(LocalResource.class); generateByNewInstance(Priority.class); generateByNewInstance(NodeId.class); @@ -1058,6 +1062,12 @@ public void testResourcePBImpl() throws Exception { } @Test + public void testResourceUtilizationPBImpl() throws Exception { + validatePBImplRecord(ResourceUtilizationPBImpl.class, + ResourceUtilizationProto.class); + } + + @Test public void testResourceRequestPBImpl() throws Exception { validatePBImplRecord(ResourceRequestPBImpl.class, ResourceRequestProto.class); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeHealthStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeHealthStatus.java index b21b880..6c608c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeHealthStatus.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeHealthStatus.java @@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.util.Records; /** @@ -90,4 +91,17 @@ public static NodeHealthStatus newInstance(boolean isNodeHealthy, @Private @Unstable public abstract void setLastHealthReportTime(long lastHealthReport); + + /** + * Get the resource utilization of the containers. + * @return resource utilization of the containers + */ + @Public + @Stable + public abstract ResourceUtilization getContainersUtilization(); + + @Private + @Unstable + public abstract void setContainersUtilization( + ResourceUtilization containersUtilization); } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeHealthStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeHealthStatusPBImpl.java index 75aa3d1..7687efc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeHealthStatusPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeHealthStatusPBImpl.java @@ -18,6 +18,11 @@ package org.apache.hadoop.yarn.server.api.records.impl.pb; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ResourceUtilization; +import org.apache.hadoop.yarn.api.records.impl.pb.ResourceUtilizationPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceUtilizationProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeHealthStatusProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeHealthStatusProtoOrBuilder; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; @@ -128,4 +133,33 @@ public void setLastHealthReportTime(long lastHealthReport) { this.builder.setLastHealthReportTime((lastHealthReport)); } + @Override + public ResourceUtilization getContainersUtilization() { + NodeHealthStatusProtoOrBuilder p = + this.viaProto ? this.proto : this.builder; + if (!p.hasContainersUtilization()) { + return null; + } + return convertFromProtoFormat(p.getContainersUtilization()); + } + + @Override + public void setContainersUtilization(ResourceUtilization containersUtilization) { + maybeInitBuilder(); + if (containersUtilization == null) { + this.builder.clearContainersUtilization(); + return; + } + this.builder + .setContainersUtilization(convertToProtoFormat(containersUtilization)); + } + + private ResourceUtilizationProto convertToProtoFormat(ResourceUtilization r) { + return ((ResourceUtilizationPBImpl) r).getProto(); + } + + private ResourceUtilizationPBImpl convertFromProtoFormat( + ResourceUtilizationProto p) { + return new ResourceUtilizationPBImpl(p); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto index 99149ac..af36fc7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto @@ -47,6 +47,7 @@ message NodeHealthStatusProto { optional bool is_node_healthy = 1; optional string health_report = 2; optional int64 last_health_report_time = 3; + optional ResourceUtilizationProto containers_utilization = 4; } message VersionProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index 3721b0e..df25419 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -77,6 +78,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider; import org.apache.hadoop.yarn.util.Records; @@ -424,11 +426,15 @@ private NodeStatus getNodeStatus(int responseId) throws IOException { nodeHealthStatus.setIsNodeHealthy(healthChecker.isHealthy()); nodeHealthStatus.setLastHealthReportTime(healthChecker .getLastHealthReportTime()); + + ResourceUtilization containersUtilization = getContainersUtilization(); + nodeHealthStatus.setContainersUtilization(containersUtilization); if (LOG.isDebugEnabled()) { LOG.debug("Node's health-status : " + nodeHealthStatus.getIsNodeHealthy() + ", " + nodeHealthStatus.getHealthReport()); } List containersStatuses = getContainerStatuses(); + NodeStatus nodeStatus = NodeStatus.newInstance(nodeId, responseId, containersStatuses, createKeepAliveApplicationList(), nodeHealthStatus); @@ -436,6 +442,18 @@ private NodeStatus getNodeStatus(int responseId) throws IOException { return nodeStatus; } + /** + * Get the aggregated utilization of the containers in this node. + * @return Resource utilization of all the containers. + */ + private ResourceUtilization getContainersUtilization() { + ContainerManagerImpl containerManager = + (ContainerManagerImpl) this.context.getContainerManager(); + ContainersMonitor containersMonitor = + containerManager.getContainersMonitor(); + return containersMonitor.getContainersUtilization(); + } + // Iterate through the NMContext and clone and get all the containers' // statuses. If it's a completed container, add into the // recentlyStoppedContainers collections. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 289d6d0..bdc9a89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java index d3e2bf2..4d69dbf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java @@ -19,10 +19,11 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor; import org.apache.hadoop.service.Service; +import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.nodemanager.ResourceView; public interface ContainersMonitor extends Service, EventHandler, ResourceView { - + public ResourceUtilization getContainersUtilization(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index b5f154d..7713eeb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -32,6 +32,7 @@ import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; @@ -41,6 +42,7 @@ import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils; import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; +import org.apache.hadoop.yarn.util.resource.ResourceUtilizations; import com.google.common.base.Preconditions; @@ -78,6 +80,8 @@ private static final long UNKNOWN_MEMORY_LIMIT = -1L; private int nodeCpuPercentageForYARN; + private ResourceUtilization containersUtilization; + public ContainersMonitorImpl(ContainerExecutor exec, AsyncDispatcher dispatcher, Context context) { super("containers-monitor"); @@ -89,6 +93,8 @@ public ContainersMonitorImpl(ContainerExecutor exec, this.containersToBeAdded = new HashMap(); this.containersToBeRemoved = new ArrayList(); this.monitoringThread = new MonitoringThread(); + + this.containersUtilization = ResourceUtilization.newInstance(0, 0, 0.0f); } @Override @@ -384,6 +390,11 @@ public void run() { containersToBeRemoved.clear(); } + // Temporary structure to calculate the total resource utilization of + // the containers + ResourceUtilization auxContainersUtilization = + ResourceUtilization.newInstance(0, 0, 0.0f); + // Now do the monitoring for the trackingContainers // Check memory usage and kill any overflowing containers long vmemUsageByAllContainers = 0; @@ -463,6 +474,18 @@ public void run() { currentPmemUsage, pmemLimit)); } + // Save utilization for this container + ResourceUtilization containerUtilization = + ResourceUtilization.newInstance((int) (currentPmemUsage >> 20), + (int) (currentVmemUsage >> 20), milliVcoresUsed / 1000.0f); + ResourceUtilizations.addTo(auxContainersUtilization, + containerUtilization); + + if (LOG.isDebugEnabled()) { + LOG.debug("Container " + containerId + " utilization is " + + containerUtilization); + } + // Add usage to container metrics if (containerMetricsEnabled) { ContainerMetrics.forContainer( @@ -542,6 +565,9 @@ public void run() { + cpuUsagePercentPerCoreByAllContainers); } + // Save the aggregated utilization of the containers + setContainersUtilization(auxContainersUtilization); + try { Thread.sleep(monitoringInterval); } catch (InterruptedException e) { @@ -614,6 +640,15 @@ public boolean isVmemCheckEnabled() { } @Override + public ResourceUtilization getContainersUtilization() { + return this.containersUtilization; + } + + public void setContainersUtilization(ResourceUtilization utilization) { + this.containersUtilization = utilization; + } + + @Override public void handle(ContainersMonitorEvent monitoringEvent) { if (!isEnabled()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java index b2ccb61..637f7c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;