diff --git hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java index c598aa0..b18fb4a 100644 --- hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java +++ hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java @@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; -import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus; +import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode @@ -191,7 +191,7 @@ public void updateNodeHeartbeatResponseForContainersDecreasing( return null; } - public QueuedContainersStatus getQueuedContainersStatus() { + public OpportunisticContainersStatus getOpportunisticContainersStatus() { return null; } diff --git hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java index 6d0ffbd..4edc216 100644 --- hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java +++ hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java @@ -29,7 +29,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; -import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus; +import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode @@ -180,7 +180,7 @@ public void updateNodeHeartbeatResponseForContainersDecreasing( return Collections.EMPTY_LIST; } - public QueuedContainersStatus getQueuedContainersStatus() { + public OpportunisticContainersStatus getOpportunisticContainersStatus() { return null; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java index 89e054b..04dd8d2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java @@ -125,10 +125,10 @@ public abstract void setIncreasedContainers( @Private @Unstable - public abstract QueuedContainersStatus getQueuedContainersStatus(); + public abstract OpportunisticContainersStatus getOpportunisticContainersStatus(); @Private @Unstable - public abstract void setQueuedContainersStatus( - QueuedContainersStatus queuedContainersStatus); + public abstract void setOpportunisticContainersStatus( + OpportunisticContainersStatus opportunisticContainersStatus); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OpportunisticContainersStatus.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OpportunisticContainersStatus.java new file mode 100644 index 0000000..8d68769 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OpportunisticContainersStatus.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * OpportunisticContainersStatus captures information pertaining to the + * state of execution of the opportunistic containers within a node. + *

+ */ +@Private +@Evolving +public abstract class OpportunisticContainersStatus { + public static OpportunisticContainersStatus newInstance() { + return Records.newRecord(OpportunisticContainersStatus.class); + } + + public abstract int getEstimatedQueueWaitTime(); + + public abstract void setEstimatedQueueWaitTime(int queueWaitTime); + + public abstract int getWaitQueueLength(); + + public abstract void setWaitQueueLength(int waitQueueLength); + + public abstract int getRunningOpportContainers(); + + public abstract void setRunningOpportContainers(int runningOpportContainers); + + public abstract long getOpportMemoryUsed(); + + public abstract void setOpportMemoryUsed(long opportMemoryUsed); + + public abstract long getOpportCoresUsed(); + + public abstract void setOpportCoresUsed(long opportCoresUsed); + + public abstract int getQueuedContainers(); + + public abstract void setQueuedContainers(int queuedContainers); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/QueuedContainersStatus.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/QueuedContainersStatus.java deleted file mode 100644 index fb567d5..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/QueuedContainersStatus.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.records; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.yarn.util.Records; - -/** - *

- * QueuedContainersStatus captures information pertaining to the - * state of execution of the Queueable containers within a node. - *

- */ -@Private -@Evolving -public abstract class QueuedContainersStatus { - public static QueuedContainersStatus newInstance() { - return Records.newRecord(QueuedContainersStatus.class); - } - - public abstract int getEstimatedQueueWaitTime(); - - public abstract void setEstimatedQueueWaitTime(int queueWaitTime); - - public abstract int getWaitQueueLength(); - - public abstract void setWaitQueueLength(int waitQueueLength); -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java index d6a1737..0a34225 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java @@ -41,9 +41,9 @@ import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeHealthStatusProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProtoOrBuilder; -import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.QueuedContainersStatusProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.OpportunisticContainersStatusProto; -import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus; +import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.api.records.NodeStatus; @@ -404,25 +404,25 @@ public synchronized void setIncreasedContainers( } @Override - public synchronized QueuedContainersStatus getQueuedContainersStatus() { - NodeStatusProtoOrBuilder p = - this.viaProto ? this.proto : this.builder; - if (!p.hasQueuedContainerStatus()) { + public synchronized OpportunisticContainersStatus + getOpportunisticContainersStatus() { + NodeStatusProtoOrBuilder p = this.viaProto ? this.proto : this.builder; + if (!p.hasOpportunisticContainersStatus()) { return null; } - return convertFromProtoFormat(p.getQueuedContainerStatus()); + return convertFromProtoFormat(p.getOpportunisticContainersStatus()); } @Override - public synchronized void setQueuedContainersStatus( - QueuedContainersStatus queuedContainersStatus) { + public synchronized void setOpportunisticContainersStatus( + OpportunisticContainersStatus opportunisticContainersStatus) { maybeInitBuilder(); - if (queuedContainersStatus == null) { - this.builder.clearQueuedContainerStatus(); + if (opportunisticContainersStatus == null) { + this.builder.clearOpportunisticContainersStatus(); return; } - this.builder.setQueuedContainerStatus( - convertToProtoFormat(queuedContainersStatus)); + this.builder.setOpportunisticContainersStatus( + convertToProtoFormat(opportunisticContainersStatus)); } private NodeIdProto convertToProtoFormat(NodeId nodeId) { @@ -468,14 +468,14 @@ private ResourceUtilizationPBImpl convertFromProtoFormat( return new ResourceUtilizationPBImpl(p); } - private QueuedContainersStatusProto convertToProtoFormat( - QueuedContainersStatus r) { - return ((QueuedContainersStatusPBImpl) r).getProto(); + private OpportunisticContainersStatusProto convertToProtoFormat( + OpportunisticContainersStatus r) { + return ((OpportunisticContainersStatusPBImpl) r).getProto(); } - private QueuedContainersStatus convertFromProtoFormat( - QueuedContainersStatusProto p) { - return new QueuedContainersStatusPBImpl(p); + private OpportunisticContainersStatus convertFromProtoFormat( + OpportunisticContainersStatusProto p) { + return new OpportunisticContainersStatusPBImpl(p); } private ContainerPBImpl convertFromProtoFormat( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OpportunisticContainersStatusPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OpportunisticContainersStatusPBImpl.java new file mode 100644 index 0000000..08f81ba --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OpportunisticContainersStatusPBImpl.java @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.records.impl.pb; + +import org.apache.hadoop.yarn.proto.YarnServerCommonProtos; +import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; + +/** + * Protocol Buffer implementation of OpportunisticContainersStatus. + */ +public class OpportunisticContainersStatusPBImpl + extends OpportunisticContainersStatus { + + private YarnServerCommonProtos.OpportunisticContainersStatusProto proto = + YarnServerCommonProtos.OpportunisticContainersStatusProto + .getDefaultInstance(); + private YarnServerCommonProtos.OpportunisticContainersStatusProto.Builder + builder = null; + private boolean viaProto = false; + + public OpportunisticContainersStatusPBImpl() { + builder = + YarnServerCommonProtos.OpportunisticContainersStatusProto.newBuilder(); + } + + public OpportunisticContainersStatusPBImpl(YarnServerCommonProtos + .OpportunisticContainersStatusProto proto) { + this.proto = proto; + viaProto = true; + } + + public YarnServerCommonProtos.OpportunisticContainersStatusProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = YarnServerCommonProtos.OpportunisticContainersStatusProto + .newBuilder(proto); + } + viaProto = false; + } + + @Override + public int getEstimatedQueueWaitTime() { + YarnServerCommonProtos.OpportunisticContainersStatusProtoOrBuilder p = + viaProto ? proto : builder; + return p.getEstimatedQueueWaitTime(); + } + + @Override + public void setEstimatedQueueWaitTime(int queueWaitTime) { + maybeInitBuilder(); + builder.setEstimatedQueueWaitTime(queueWaitTime); + } + + @Override + public int getWaitQueueLength() { + YarnServerCommonProtos.OpportunisticContainersStatusProtoOrBuilder p = + viaProto ? proto : builder; + return p.getWaitQueueLength(); + } + + @Override + public void setWaitQueueLength(int waitQueueLength) { + maybeInitBuilder(); + builder.setWaitQueueLength(waitQueueLength); + } + + @Override + public int getRunningOpportContainers() { + YarnServerCommonProtos.OpportunisticContainersStatusProtoOrBuilder p = + viaProto ? proto : builder; + return p.getRunningOpportContainers(); + } + + @Override + public void setRunningOpportContainers(int runningOpportContainers) { + maybeInitBuilder(); + builder.setRunningOpportContainers(runningOpportContainers); + } + + @Override + public long getOpportMemoryUsed() { + YarnServerCommonProtos.OpportunisticContainersStatusProtoOrBuilder p = + viaProto ? proto : builder; + return p.getOpportMemoryUsed(); + } + + @Override + public void setOpportMemoryUsed(long opportMemoryUsed) { + maybeInitBuilder(); + builder.setOpportMemoryUsed(opportMemoryUsed); + } + + @Override + public long getOpportCoresUsed() { + YarnServerCommonProtos.OpportunisticContainersStatusProtoOrBuilder p = + viaProto ? proto : builder; + return p.getOpportCoresUsed(); + } + + @Override + public void setOpportCoresUsed(long opportCoresUsed) { + maybeInitBuilder(); + builder.setOpportCoresUsed(opportCoresUsed); + } + + @Override + public int getQueuedContainers() { + YarnServerCommonProtos.OpportunisticContainersStatusProtoOrBuilder p = + viaProto ? proto : builder; + return p.getQueuedContainers(); + } + + @Override + public void setQueuedContainers(int queuedContainers) { + maybeInitBuilder(); + builder.setQueuedContainers(queuedContainers); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/QueuedContainersStatusPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/QueuedContainersStatusPBImpl.java deleted file mode 100644 index 16b80dd..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/QueuedContainersStatusPBImpl.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.records.impl.pb; - -import org.apache.hadoop.yarn.proto.YarnServerCommonProtos; -import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus; - -/** - * Protocol Buffer implementation of QueuedContainersStatus. - */ -public class QueuedContainersStatusPBImpl extends QueuedContainersStatus { - - private YarnServerCommonProtos.QueuedContainersStatusProto proto = - YarnServerCommonProtos.QueuedContainersStatusProto.getDefaultInstance(); - private YarnServerCommonProtos.QueuedContainersStatusProto.Builder builder = - null; - private boolean viaProto = false; - - public QueuedContainersStatusPBImpl() { - builder = YarnServerCommonProtos.QueuedContainersStatusProto.newBuilder(); - } - - public QueuedContainersStatusPBImpl(YarnServerCommonProtos - .QueuedContainersStatusProto proto) { - this.proto = proto; - viaProto = true; - } - - public YarnServerCommonProtos.QueuedContainersStatusProto getProto() { - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = - YarnServerCommonProtos.QueuedContainersStatusProto.newBuilder(proto); - } - viaProto = false; - } - - @Override - public int getEstimatedQueueWaitTime() { - YarnServerCommonProtos.QueuedContainersStatusProtoOrBuilder p = - viaProto ? proto : builder; - return p.getEstimatedQueueWaitTime(); - } - - @Override - public void setEstimatedQueueWaitTime(int queueWaitTime) { - maybeInitBuilder(); - builder.setEstimatedQueueWaitTime(queueWaitTime); - } - - @Override - public int getWaitQueueLength() { - YarnServerCommonProtos.QueuedContainersStatusProtoOrBuilder p = - viaProto ? proto : builder; - return p.getWaitQueueLength(); - } - - @Override - public void setWaitQueueLength(int waitQueueLength) { - maybeInitBuilder(); - builder.setWaitQueueLength(waitQueueLength); - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto index c23d557..ef28126 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto @@ -39,12 +39,16 @@ message NodeStatusProto { optional ResourceUtilizationProto containers_utilization = 6; optional ResourceUtilizationProto node_utilization = 7; repeated ContainerProto increased_containers = 8; - optional QueuedContainersStatusProto queued_container_status = 9; + optional OpportunisticContainersStatusProto opportunistic_containers_status = 9; } -message QueuedContainersStatusProto { +message OpportunisticContainersStatusProto { optional int32 estimated_queue_wait_time = 1; optional int32 wait_queue_length = 2; + optional int32 running_opport_containers = 3; + optional int64 opport_memory_used = 4; + optional int64 opport_cores_used = 5; + optional int32 queued_containers = 6; } message MasterKeyProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java index 9f4b436..204be22 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java @@ -48,7 +48,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RegisterNodeManagerRequestPBImpl; import org.apache.hadoop.yarn.server.api.records.NodeStatus; -import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus; +import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; import org.apache.hadoop.yarn.util.Records; import org.junit.Assert; import org.junit.Test; @@ -146,11 +146,11 @@ public void testNodeHeartBeatRequest() throws IOException { Records.newRecord(NodeHeartbeatRequest.class); NodeStatus nodeStatus = Records.newRecord(NodeStatus.class); - QueuedContainersStatus queuedContainersStatus = Records.newRecord - (QueuedContainersStatus.class); - queuedContainersStatus.setEstimatedQueueWaitTime(123); - queuedContainersStatus.setWaitQueueLength(321); - nodeStatus.setQueuedContainersStatus(queuedContainersStatus); + OpportunisticContainersStatus opportunisticContainersStatus = Records.newRecord + (OpportunisticContainersStatus.class); + opportunisticContainersStatus.setEstimatedQueueWaitTime(123); + opportunisticContainersStatus.setWaitQueueLength(321); + nodeStatus.setOpportunisticContainersStatus(opportunisticContainersStatus); record.setNodeStatus(nodeStatus); NodeHeartbeatRequestPBImpl pb = new @@ -159,9 +159,9 @@ public void testNodeHeartBeatRequest() throws IOException { Assert.assertEquals(123, pb.getNodeStatus() - .getQueuedContainersStatus().getEstimatedQueueWaitTime()); + .getOpportunisticContainersStatus().getEstimatedQueueWaitTime()); Assert.assertEquals(321, - pb.getNodeStatus().getQueuedContainersStatus().getWaitQueueLength()); + pb.getNodeStatus().getOpportunisticContainersStatus().getWaitQueueLength()); } @Test diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index f692bf1..6d73ac7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -75,7 +75,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.UnRegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit; -import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus; +import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; @@ -465,16 +465,24 @@ protected NodeStatus getNodeStatus(int responseId) throws IOException { createKeepAliveApplicationList(), nodeHealthStatus, containersUtilization, nodeUtilization, increasedContainers); - nodeStatus.setQueuedContainersStatus(getQueuedContainerStatus()); + nodeStatus.setOpportunisticContainersStatus( + getOpportunisticContainersStatus()); return nodeStatus; } - private QueuedContainersStatus getQueuedContainerStatus() { - QueuedContainersStatus status = QueuedContainersStatus.newInstance(); - status.setWaitQueueLength( - this.context.getQueuingContext().getQueuedContainers().size()); + /** + * Get the status of the OPPORTUNISTIC containers. + * @return the status of the OPPORTUNISTIC containers. + */ + private OpportunisticContainersStatus getOpportunisticContainersStatus() { + OpportunisticContainersStatus status = + this.context.getContainerManager().getOpportunisticContainersStatus(); + if (status == null) { + status = OpportunisticContainersStatus.newInstance(); + } return status; } + /** * Get the aggregated utilization of the containers in this node. * @return Resource utilization of all the containers. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManager.java index 0da02b3..1cbb8c7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManager.java @@ -22,6 +22,7 @@ import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit; +import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; import org.apache.hadoop.yarn.server.nodemanager.ContainerManagerEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor .ContainersMonitor; @@ -35,6 +36,8 @@ ContainersMonitor getContainersMonitor(); + OpportunisticContainersStatus getOpportunisticContainersStatus(); + void updateQueuingLimit(ContainerQueuingLimit queuingLimit); void setBlockNewContainerRequests(boolean blockNewContainerRequests); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index e8c2b75..76933ec 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -88,6 +88,7 @@ import org.apache.hadoop.yarn.security.NMTokenIdentifier; import org.apache.hadoop.yarn.server.api.ContainerType; import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit; +import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent; import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedContainersEvent; import org.apache.hadoop.yarn.server.nodemanager.CMgrDecreaseContainersResourceEvent; @@ -1521,8 +1522,13 @@ protected boolean isServiceStopped() { } @Override + public OpportunisticContainersStatus getOpportunisticContainersStatus() { + return null; + } + + @Override public void updateQueuingLimit(ContainerQueuingLimit queuingLimit) { - LOG.trace("Implementation does not support queuing of Containers !!"); + LOG.trace("Implementation does not support queuing of Containers!!"); } @SuppressWarnings("unchecked") diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java index 5d2f4d4..590ba51 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.security.NMTokenIdentifier; import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit; +import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; @@ -80,10 +81,14 @@ private ConcurrentMap allocatedOpportunisticContainers; + private long allocatedMemoryOpportunistic; + private long allocatedVCoresOpportunistic; + private Queue queuedGuaranteedContainers; private Queue queuedOpportunisticContainers; private Set opportunisticContainersToKill; + private final OpportunisticContainersStatus opportunisticContainersStatus; private final ContainerQueuingLimit queuingLimit; public QueuingContainerManagerImpl(Context context, ContainerExecutor exec, @@ -93,10 +98,14 @@ public QueuingContainerManagerImpl(Context context, ContainerExecutor exec, dirsHandler); this.allocatedGuaranteedContainers = new ConcurrentHashMap<>(); this.allocatedOpportunisticContainers = new ConcurrentHashMap<>(); + this.allocatedMemoryOpportunistic = 0; + this.allocatedVCoresOpportunistic = 0; this.queuedGuaranteedContainers = new ConcurrentLinkedQueue<>(); this.queuedOpportunisticContainers = new ConcurrentLinkedQueue<>(); this.opportunisticContainersToKill = Collections.synchronizedSet( new HashSet()); + this.opportunisticContainersStatus = + OpportunisticContainersStatus.newInstance(); this.queuingLimit = ContainerQueuingLimit.newInstance(); } @@ -196,6 +205,8 @@ private void startAllocatedContainer( } else { allocatedOpportunisticContainers.put(pti.getContainerId(), allocatedContainerInfo); + allocatedMemoryOpportunistic += pti.getPmemLimit(); + allocatedVCoresOpportunistic += pti.getCpuVcores(); } getContainersMonitor().increaseContainersAllocation(pti); @@ -267,6 +278,8 @@ private void removeAllocatedContainer(ContainerId containerId) { if (contToRemove != null) { getContainersMonitor().decreaseContainersAllocation(contToRemove .getPti()); + allocatedMemoryOpportunistic -= contToRemove.getPti().getPmemLimit(); + allocatedVCoresOpportunistic -= contToRemove.getPti().getCpuVcores(); } } @@ -557,6 +570,22 @@ public void handle(ApplicationEvent event) { } @Override + public OpportunisticContainersStatus getOpportunisticContainersStatus() { + opportunisticContainersStatus.setWaitQueueLength( + queuedGuaranteedContainers.size() + + queuedOpportunisticContainers.size()); + opportunisticContainersStatus + .setRunningOpportContainers(allocatedOpportunisticContainers.size()); + opportunisticContainersStatus + .setOpportMemoryUsed(allocatedMemoryOpportunistic); + opportunisticContainersStatus + .setOpportCoresUsed(allocatedVCoresOpportunistic); + opportunisticContainersStatus + .setQueuedContainers(queuedOpportunisticContainers.size()); + return opportunisticContainersStatus; + } + + @Override public void updateQueuingLimit(ContainerQueuingLimit limit) { this.queuingLimit.setMaxQueueLength(limit.getMaxQueueLength()); // TODO: Include wait time as well once it is implemented diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java index 10e2afa..6d95b56 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java @@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; -import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus; +import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; /** * Node managers information on available resources @@ -170,7 +170,7 @@ public void updateNodeHeartbeatResponseForContainersDecreasing( public List pullNewlyIncreasedContainers(); - QueuedContainersStatus getQueuedContainersStatus(); + OpportunisticContainersStatus getOpportunisticContainersStatus(); long getUntrackedTimeStamp(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java index 375b4cf..235460d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java @@ -61,7 +61,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; -import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus; +import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics; import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEvent; @@ -134,7 +134,7 @@ private ResourceUtilization nodeUtilization; /* Container Queue Information for the node.. Used by Distributed Scheduler */ - private QueuedContainersStatus queuedContainersStatus; + private OpportunisticContainersStatus opportunisticContainersStatus; private final ContainerAllocationExpirer containerAllocationExpirer; /* set of containers that have just launched */ @@ -1164,7 +1164,7 @@ public void transition(RMNodeImpl rmNode, RMNodeEvent event) { public NodeState transition(RMNodeImpl rmNode, RMNodeEvent event) { RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event; - rmNode.setQueuedContainersStatus(statusEvent.getContainerQueueInfo()); + rmNode.setOpportunisticContainersStatus(statusEvent.getContainerQueueInfo()); NodeHealthStatus remoteNodeHealthStatus = updateRMNodeFromStatusEvents( rmNode, statusEvent); NodeState initialState = rmNode.getState(); @@ -1460,22 +1460,21 @@ public Resource getOriginalTotalCapability() { return this.originalTotalCapability; } - public QueuedContainersStatus getQueuedContainersStatus() { + public OpportunisticContainersStatus getOpportunisticContainersStatus() { this.readLock.lock(); try { - return this.queuedContainersStatus; + return this.opportunisticContainersStatus; } finally { this.readLock.unlock(); } } - public void setQueuedContainersStatus(QueuedContainersStatus - queuedContainersStatus) { + public void setOpportunisticContainersStatus(OpportunisticContainersStatus opportunisticContainersStatus) { this.writeLock.lock(); try { - this.queuedContainersStatus = queuedContainersStatus; + this.opportunisticContainersStatus = opportunisticContainersStatus; } finally { this.writeLock.unlock(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java index 5eeaabe..a7ba096 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java @@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; -import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus; +import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.api.records.NodeStatus; @@ -80,8 +80,8 @@ public ResourceUtilization getNodeUtilization() { return this.logAggregationReportsForApps; } - public QueuedContainersStatus getContainerQueueInfo() { - return this.nodeStatus.getQueuedContainersStatus(); + public OpportunisticContainersStatus getContainerQueueInfo() { + return this.nodeStatus.getOpportunisticContainersStatus(); } public void setLogAggregationReportsForApps( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java index 017a256..add8c95 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java @@ -24,7 +24,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; -import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus; +import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; import org.apache.hadoop.yarn.server.resourcemanager.ClusterMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -179,11 +179,11 @@ public void removeNode(RMNode removedRMNode) { @Override public void updateNode(RMNode rmNode) { LOG.debug("Node update event from: " + rmNode.getNodeID()); - QueuedContainersStatus queuedContainersStatus = - rmNode.getQueuedContainersStatus(); + OpportunisticContainersStatus opportunisticContainersStatus = + rmNode.getOpportunisticContainersStatus(); int estimatedQueueWaitTime = - queuedContainersStatus.getEstimatedQueueWaitTime(); - int waitQueueLength = queuedContainersStatus.getWaitQueueLength(); + opportunisticContainersStatus.getEstimatedQueueWaitTime(); + int waitQueueLength = opportunisticContainersStatus.getWaitQueueLength(); // Add nodes to clusterNodes. If estimatedQueueTime is -1, ignore node // UNLESS comparator is based on queue length. synchronized (this.clusterNodes) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java index 7063421..45bb453 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java @@ -63,7 +63,7 @@ protected void render(Block html) { ResourceScheduler sched = rm.getResourceScheduler(); String type = $(NODE_STATE); String labelFilter = $(NODE_LABEL, CommonNodeLabelsManager.ANY).trim(); - TBODY> tbody = + Hamlet.TR>> trbody = html.table("#nodes").thead().tr() .th(".nodelabels", "Node Labels") .th(".rack", "Rack") @@ -71,13 +71,30 @@ protected void render(Block html) { .th(".nodeaddress", "Node Address") .th(".nodehttpaddress", "Node HTTP Address") .th(".lastHealthUpdate", "Last health-update") - .th(".healthReport", "Health-report") - .th(".containers", "Containers") - .th(".mem", "Mem Used") - .th(".mem", "Mem Avail") - .th(".vcores", "VCores Used") - .th(".vcores", "VCores Avail") - .th(".nodeManagerVersion", "Version")._()._().tbody(); + .th(".healthReport", "Health-report"); + + // Here check if opportunistic containers are enabled. + if (true) { + trbody.th(".containers", "Containers") + .th(".mem", "Mem Used") + .th(".mem", "Mem Avail") + .th(".vcores", "VCores Used") + .th(".vcores", "VCores Avail"); + } else { + trbody.th(".containers", "Running Containers (G)") + .th(".mem", "Mem Used (G)") + .th(".mem", "Mem Avail (G)") + .th(".vcores", "VCores Used (G)") + .th(".vcores", "VCores Avail (G)") + .th(".containers", "Running Containers (O)") + .th(".mem", "Mem Used (O)") + .th(".vcores", "VCores Used (O)") + .th(".containers", "Queued Containers"); + } + + TBODY> tbody = + trbody.th(".nodeManagerVersion", "Version")._()._().tbody(); + NodeState stateFilter = null; if (type != null && !type.isEmpty()) { stateFilter = NodeState.valueOf(StringUtils.toUpperCase(type)); @@ -153,7 +170,23 @@ protected void render(Block html) { .append("\",\"").append(String.valueOf(info.getUsedVirtualCores())) .append("\",\"") .append(String.valueOf(info.getAvailableVirtualCores())) - .append("\",\"").append(ni.getNodeManagerVersion()) + .append("\",\""); + + // If opportunistic containers are enabled. + if (true) { + nodeTableData + .append(String.valueOf(info.getNumRunningOpportContainers())) + .append("\",\"").append("
") + .append(StringUtils.byteDesc(usedMemory * BYTES_IN_MB)) + .append("\",\"") + .append(String.valueOf(info.getUsedVirtualCores())) + .append("\",\"") + .append(String.valueOf(info.getNumRunningOpportContainers())) + .append("\",\""); + } + + nodeTableData.append(ni.getNodeManagerVersion()) .append("\"],\n"); } if (nodeTableData.charAt(nodeTableData.length() - 2) == ',') { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java index 4a6aa4b..8d94961 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; @@ -49,6 +50,10 @@ protected long availMemoryMB; protected long usedVirtualCores; protected long availableVirtualCores; + protected int numRunningOpportContainers; + protected long usedMemoryOpportMB; + protected long usedVirtualCoresOpport; + protected int numQueuedContainers; protected ArrayList nodeLabels = new ArrayList(); protected ResourceUtilizationInfo resourceUtilization; @@ -76,7 +81,15 @@ public NodeInfo(RMNode ni, ResourceScheduler sched) { this.lastHealthUpdate = ni.getLastHealthReportTime(); this.healthReport = String.valueOf(ni.getHealthReport()); this.version = ni.getNodeManagerVersion(); - + + // Status of opportunistic containers. + OpportunisticContainersStatus opportStatus = + ni.getOpportunisticContainersStatus(); + this.numRunningOpportContainers = opportStatus.getRunningOpportContainers(); + this.usedMemoryOpportMB = opportStatus.getOpportMemoryUsed(); + this.usedVirtualCoresOpport = opportStatus.getOpportCoresUsed(); + this.numQueuedContainers = opportStatus.getQueuedContainers(); + // add labels Set labelSet = ni.getNodeLabels(); if (labelSet != null) { @@ -140,6 +153,22 @@ public long getAvailableVirtualCores() { return this.availableVirtualCores; } + public int getNumRunningOpportContainers() { + return numRunningOpportContainers; + } + + public long getUsedMemoryOpport() { + return usedMemoryOpportMB; + } + + public long getUsedVirtualCoresOpport() { + return usedVirtualCoresOpport; + } + + public int getNumQueuedContainers() { + return numQueuedContainers; + } + public ArrayList getNodeLabels() { return this.nodeLabels; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index 5a89e54..7959e74 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -35,7 +35,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; -import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus; +import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; @@ -260,7 +260,7 @@ public ResourceUtilization getNodeUtilization() { return this.nodeUtilization; } - public QueuedContainersStatus getQueuedContainersStatus() { + public OpportunisticContainersStatus getOpportunisticContainersStatus() { return null; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/TestNodeQueueLoadMonitor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/TestNodeQueueLoadMonitor.java index 5f63923..dfd21ff 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/TestNodeQueueLoadMonitor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/TestNodeQueueLoadMonitor.java @@ -20,7 +20,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit; -import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus; +import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.junit.Assert; import org.junit.Test; @@ -183,13 +183,13 @@ private RMNode createRMNode(String host, int port, RMNode node1 = Mockito.mock(RMNode.class); NodeId nID1 = new FakeNodeId(host, port); Mockito.when(node1.getNodeID()).thenReturn(nID1); - QueuedContainersStatus status1 = - Mockito.mock(QueuedContainersStatus.class); + OpportunisticContainersStatus status1 = + Mockito.mock(OpportunisticContainersStatus.class); Mockito.when(status1.getEstimatedQueueWaitTime()) .thenReturn(waitTime); Mockito.when(status1.getWaitQueueLength()) .thenReturn(queueLength); - Mockito.when(node1.getQueuedContainersStatus()).thenReturn(status1); + Mockito.when(node1.getOpportunisticContainersStatus()).thenReturn(status1); return node1; } }