diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 9a1eb54..29f4345 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1275,7 +1275,17 @@ private static void addDeprecatedKeys() {
public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE =
false;
-
+ /** Overallocation (= allocation based on utilization) configs. */
+ public static final String NM_OVERALLOCATION_ALLOCATION_THRESHOLD =
+ NM_PREFIX + "overallocation.allocation-threshold";
+ public static final float DEFAULT_NM_OVERALLOCATION_ALLOCATION_THRESHOLD
+ = 0f;
+ @Private
+ public static final float MAX_NM_OVERALLOCATION_ALLOCATION_THRESHOLD = 0.95f;
+ public static final String NM_OVERALLOCATION_PREEMPTION_THRESHOLD =
+ NM_PREFIX + "overallocation.preemption-threshold";
+ public static final float DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD
+ = 0f;
/**
* Interval of time the linux container executor should try cleaning up
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 49cced6..a3afde2 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1447,6 +1447,27 @@
+ The extent of over-allocation (container-allocation based on
+ current utilization instead of prior allocation) allowed on this node,
+ expressed as a float between 0 and 0.95. By default, over-allocation is
+ turned off (value = 0). When turned on, the node allows running
+ OPPORTUNISTIC containers when the aggregate utilization is under the
+ value specified here multiplied by the node's advertised capacity.
+
+ yarn.nodemanager.overallocation.allocation-threshold
+ 0f
+
+
+
+ When a node is over-allocated to improve utilization by
+ running OPPORTUNISTIC containers, this config captures the utilization
+ beyond which OPPORTUNISTIC containers should start getting preempted.
+
+ yarn.nodemanager.overallocation.preemption-threshold
+ 1
+
+
+
This configuration setting determines the capabilities
assigned to docker containers when they are launched. While these may not
be case-sensitive from a docker perspective, it is best to keep these
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
index 836cd4b..77e8b93 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
@@ -60,6 +60,19 @@ public static NodeStatus newInstance(NodeId nodeId, int responseId,
ResourceUtilization containersUtilization,
ResourceUtilization nodeUtilization,
List increasedContainers) {
+ return newInstance(nodeId, responseId, containerStatuses,
+ keepAliveApplications, nodeHealthStatus, containersUtilization,
+ nodeUtilization, increasedContainers, null);
+ }
+
+ public static NodeStatus newInstance(NodeId nodeId, int responseId,
+ List containerStatuses,
+ List keepAliveApplications,
+ NodeHealthStatus nodeHealthStatus,
+ ResourceUtilization containersUtilization,
+ ResourceUtilization nodeUtilization,
+ List increasedContainers,
+ OverAllocationInfo overAllocationInfo) {
NodeStatus nodeStatus = Records.newRecord(NodeStatus.class);
nodeStatus.setResponseId(responseId);
nodeStatus.setNodeId(nodeId);
@@ -69,12 +82,17 @@ public static NodeStatus newInstance(NodeId nodeId, int responseId,
nodeStatus.setContainersUtilization(containersUtilization);
nodeStatus.setNodeUtilization(nodeUtilization);
nodeStatus.setIncreasedContainers(increasedContainers);
+ nodeStatus.setOverAllocationInfo(overAllocationInfo);
return nodeStatus;
}
public abstract NodeId getNodeId();
public abstract int getResponseId();
-
+
+ public abstract OverAllocationInfo getOverAllocationInfo();
+ public abstract void setOverAllocationInfo(
+ OverAllocationInfo overAllocationInfo);
+
public abstract List getContainersStatuses();
public abstract void setContainersStatuses(
List containersStatuses);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OverAllocationInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OverAllocationInfo.java
new file mode 100644
index 0000000..77952bf
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OverAllocationInfo.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.server.api.records.impl.pb
+ .OverAllocationInfoPBImpl;
+
+/**
+ * Captures information on how aggressively the scheduler can over-allocate
+ * OPPORTUNISTIC containers on a node. This is node-specific, and is sent on
+ * the wire on each heartbeat.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class OverAllocationInfo {
+ public static OverAllocationInfo newInstance(
+ ResourceThresholds overAllocationThresholds) {
+ OverAllocationInfo info = new OverAllocationInfoPBImpl();
+ info.setOverAllocationThreshold(overAllocationThresholds);
+ return info;
+ }
+
+ public abstract ResourceThresholds getOverAllocationThresholds();
+
+ public abstract void setOverAllocationThreshold(
+ ResourceThresholds resourceThresholds);
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java
new file mode 100644
index 0000000..d57706a
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.server.api.records.impl.pb.ResourceThresholdsPBImpl;
+
+/**
+ * Captures resource thresholds to be used for allocation and preemption
+ * when over-allocation is turned on.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class ResourceThresholds {
+ public static ResourceThresholds newInstance(float threshold) {
+ ResourceThresholds thresholds = new ResourceThresholdsPBImpl();
+ thresholds.setMemoryThreshold(threshold);
+ thresholds.setCpuThreshold(threshold);
+ return thresholds;
+ }
+
+ public abstract float getMemoryThreshold();
+
+ public abstract float getCpuThreshold();
+
+ public abstract void setMemoryThreshold(float memoryThreshold);
+
+ public abstract void setCpuThreshold(float cpuThreshold);
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
index 8dd4832..ebe068b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
@@ -40,9 +40,11 @@
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeHealthStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.OverAllocationInfoProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceUtilizationProto;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
public class NodeStatusPBImpl extends NodeStatus {
NodeStatusProto proto = NodeStatusProto.getDefaultInstance();
@@ -50,6 +52,7 @@
boolean viaProto = false;
private NodeId nodeId = null;
+ private OverAllocationInfo overAllocationInfo = null;
private List containers = null;
private NodeHealthStatus nodeHealthStatus = null;
private List keepAliveApplications = null;
@@ -75,6 +78,10 @@ private synchronized void mergeLocalToBuilder() {
if (this.nodeId != null) {
builder.setNodeId(convertToProtoFormat(this.nodeId));
}
+ if (this.overAllocationInfo != null) {
+ builder.setOverAllocationInfo(
+ convertToProtoFormat(this.overAllocationInfo));
+ }
if (this.containers != null) {
addContainersToProto();
}
@@ -224,6 +231,30 @@ public synchronized int getResponseId() {
NodeStatusProtoOrBuilder p = viaProto ? proto : builder;
return p.getResponseId();
}
+
+ @Override
+ public synchronized OverAllocationInfo getOverAllocationInfo() {
+ NodeStatusProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.overAllocationInfo != null) {
+ return this.overAllocationInfo;
+ }
+ if (!p.hasOverAllocationInfo()) {
+ return null;
+ }
+ this.overAllocationInfo = convertFromProtoFormat(p.getOverAllocationInfo());
+ return this.overAllocationInfo;
+ }
+
+ @Override
+ public synchronized void setOverAllocationInfo(
+ OverAllocationInfo overAllocationInfo) {
+ maybeInitBuilder();
+ if (this.overAllocationInfo == null) {
+ builder.clearOverAllocationInfo();
+ }
+ this.overAllocationInfo = overAllocationInfo;
+ }
+
@Override
public synchronized void setResponseId(int responseId) {
maybeInitBuilder();
@@ -403,11 +434,21 @@ public synchronized void setIncreasedContainers(
private NodeIdProto convertToProtoFormat(NodeId nodeId) {
return ((NodeIdPBImpl)nodeId).getProto();
}
-
+
private NodeId convertFromProtoFormat(NodeIdProto proto) {
return new NodeIdPBImpl(proto);
}
+ private static OverAllocationInfoProto convertToProtoFormat(
+ OverAllocationInfo overAllocationInfo) {
+ return ((OverAllocationInfoPBImpl)overAllocationInfo).getProto();
+ }
+
+ private static OverAllocationInfo convertFromProtoFormat(
+ OverAllocationInfoProto proto) {
+ return new OverAllocationInfoPBImpl(proto);
+ }
+
private NodeHealthStatusProto convertToProtoFormat(
NodeHealthStatus healthStatus) {
return ((NodeHealthStatusPBImpl) healthStatus).getProto();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OverAllocationInfoPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OverAllocationInfoPBImpl.java
new file mode 100644
index 0000000..5599328
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OverAllocationInfoPBImpl.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api.records.impl.pb;
+
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.OverAllocationInfoProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.OverAllocationInfoProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.ResourceThresholdsProto;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+
+public class OverAllocationInfoPBImpl extends OverAllocationInfo {
+ private OverAllocationInfoProto proto =
+ OverAllocationInfoProto.getDefaultInstance();
+ private OverAllocationInfoProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private ResourceThresholds overAllocationThresholds = null;
+
+ public OverAllocationInfoPBImpl() {
+ builder = OverAllocationInfoProto.newBuilder();
+ }
+
+ public OverAllocationInfoPBImpl(OverAllocationInfoProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public synchronized OverAllocationInfoProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private synchronized void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private synchronized void mergeLocalToBuilder() {
+ if (overAllocationThresholds != null) {
+ builder.setOverAllocationThresholds(
+ convertToProtoFormat(overAllocationThresholds));
+ }
+ }
+
+ private synchronized void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = OverAllocationInfoProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public synchronized ResourceThresholds getOverAllocationThresholds() {
+ OverAllocationInfoProtoOrBuilder p = viaProto ? proto : builder;
+ if (overAllocationThresholds != null) {
+ return overAllocationThresholds;
+ }
+ if (!p.hasOverAllocationThresholds()) {
+ return null;
+ }
+ overAllocationThresholds =
+ convertFromProtoFormat(p.getOverAllocationThresholds());
+ return overAllocationThresholds;
+ }
+
+ @Override
+ public synchronized void setOverAllocationThreshold(
+ ResourceThresholds resourceThresholds) {
+ maybeInitBuilder();
+ if (this.overAllocationThresholds != null) {
+ builder.clearOverAllocationThresholds();
+ }
+ this.overAllocationThresholds = resourceThresholds;
+ }
+
+ private static ResourceThresholdsProto convertToProtoFormat(
+ ResourceThresholds overAllocationThresholds) {
+ return ((ResourceThresholdsPBImpl) overAllocationThresholds).getProto();
+ }
+
+ private static ResourceThresholds convertFromProtoFormat(
+ ResourceThresholdsProto overAllocationThresholdsProto) {
+ return new ResourceThresholdsPBImpl(overAllocationThresholdsProto);
+ }
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceThresholdsPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceThresholdsPBImpl.java
new file mode 100644
index 0000000..26a307a
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceThresholdsPBImpl.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.records.impl.pb;
+
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.ResourceThresholdsProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.ResourceThresholdsProtoOrBuilder;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+
+public class ResourceThresholdsPBImpl extends ResourceThresholds{
+ private ResourceThresholdsProto proto =
+ ResourceThresholdsProto.getDefaultInstance();
+ private ResourceThresholdsProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public ResourceThresholdsPBImpl() {
+ builder = ResourceThresholdsProto.newBuilder();
+ }
+
+ public ResourceThresholdsPBImpl(ResourceThresholdsProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public synchronized ResourceThresholdsProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private synchronized void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private synchronized void mergeLocalToBuilder() {
+ /*
+ * Right now, we have only memory and cpu thresholds that are floats.
+ * This is a no-op until we add other non-static fields to the proto.
+ */
+ }
+
+ private synchronized void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ResourceThresholdsProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public synchronized float getMemoryThreshold() {
+ ResourceThresholdsProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getMemory();
+ }
+
+ @Override
+ public synchronized float getCpuThreshold() {
+ ResourceThresholdsProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getCpu();
+ }
+
+ @Override
+ public synchronized void setMemoryThreshold(float memoryThreshold) {
+ maybeInitBuilder();
+ builder.setMemory(memoryThreshold);
+ }
+
+ @Override
+ public synchronized void setCpuThreshold(float cpuThreshold) {
+ maybeInitBuilder();
+ builder.setCpu(cpuThreshold);
+ }
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
index 77064a0..30ef97f 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
@@ -39,6 +39,7 @@ message NodeStatusProto {
optional ResourceUtilizationProto containers_utilization = 6;
optional ResourceUtilizationProto node_utilization = 7;
repeated ContainerProto increased_containers = 8;
+ optional OverAllocationInfoProto over_allocation_info = 9;
}
message MasterKeyProto {
@@ -57,3 +58,11 @@ message VersionProto {
optional int32 minor_version = 2;
}
+message OverAllocationInfoProto {
+ optional ResourceThresholdsProto over_allocation_thresholds = 1;
+}
+
+message ResourceThresholdsProto {
+ optional float memory = 1 [default = 0];
+ optional float cpu = 2 [default = 0];
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
index 9c2d1fb..39a4c90 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
@@ -29,6 +29,7 @@
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
@@ -87,4 +88,8 @@
ConcurrentLinkedQueue
getLogAggregationStatusForApps();
+
+ boolean isOverAllocationEnabled();
+
+ OverAllocationInfo getOverAllocationInfo();
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index a9a5411..fcd3f17 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -59,6 +59,7 @@
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -458,6 +459,7 @@ public void run() {
private boolean isDecommissioned = false;
private final ConcurrentLinkedQueue
logAggregationReportForApps;
+ private OverAllocationInfo overAllocationInfo;
public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInNM nmTokenSecretManager,
@@ -585,6 +587,20 @@ public void setSystemCrendentialsForApps(
getLogAggregationStatusForApps() {
return this.logAggregationReportForApps;
}
+
+ @Override
+ public boolean isOverAllocationEnabled() {
+ return getOverAllocationInfo() != null;
+ }
+
+ @Override
+ public OverAllocationInfo getOverAllocationInfo() {
+ return this.overAllocationInfo;
+ }
+
+ public void setOverAllocationInfo(OverAllocationInfo overAllocationInfo) {
+ this.overAllocationInfo = overAllocationInfo;
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 5806731..5646929 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -437,8 +437,9 @@ protected NodeStatus getNodeStatus(int responseId) throws IOException {
= getIncreasedContainers();
NodeStatus nodeStatus =
NodeStatus.newInstance(nodeId, responseId, containersStatuses,
- createKeepAliveApplicationList(), nodeHealthStatus,
- containersUtilization, nodeUtilization, increasedContainers);
+ createKeepAliveApplicationList(), nodeHealthStatus,
+ containersUtilization, nodeUtilization, increasedContainers,
+ this.context.getOverAllocationInfo());
return nodeStatus;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 446e7a1..13b30e4 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -35,8 +35,11 @@
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
@@ -83,6 +86,8 @@
private ResourceUtilization containersUtilization;
+ private ResourceThresholds overAllocationPreemptionThresholds;
+
private volatile boolean stopped = false;
public ContainersMonitorImpl(ContainerExecutor exec,
@@ -158,6 +163,13 @@ protected void serviceInit(Configuration conf) throws Exception {
LOG.info("Physical memory check enabled: " + pmemCheckEnabled);
LOG.info("Virtual memory check enabled: " + vmemCheckEnabled);
+ initializeOverAllocation(conf);
+ if (context.isOverAllocationEnabled()) {
+ pmemCheckEnabled = true;
+ LOG.info("Force enabling physical memory checks because " +
+ "overallocation is enabled");
+ }
+
containersMonitorEnabled = isEnabled();
LOG.info("ContainersMonitor enabled: " + containersMonitorEnabled);
@@ -191,6 +203,28 @@ protected void serviceInit(Configuration conf) throws Exception {
super.serviceInit(conf);
}
+ private void initializeOverAllocation(Configuration conf) {
+ float overAllocationTreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_ALLOCATION_THRESHOLD,
+ YarnConfiguration.DEFAULT_NM_OVERALLOCATION_ALLOCATION_THRESHOLD);
+ overAllocationTreshold = Math.min(overAllocationTreshold,
+ YarnConfiguration.MAX_NM_OVERALLOCATION_ALLOCATION_THRESHOLD);
+ overAllocationTreshold = Math.max(0, overAllocationTreshold);
+
+ if (overAllocationTreshold > 0f) {
+ ((NodeManager.NMContext) context).setOverAllocationInfo(
+ OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(overAllocationTreshold)));
+
+ float preemptionThreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD,
+ YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD);
+
+ this.overAllocationPreemptionThresholds =
+ ResourceThresholds.newInstance(preemptionThreshold);
+ }
+ }
+
private boolean isEnabled() {
if (resourceCalculatorPlugin == null) {
LOG.info("ResourceCalculatorPlugin is unavailable on this system. "
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index 9bc23f6..fc32253 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
@@ -60,6 +60,7 @@
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeResourceMonitor;
@@ -674,6 +675,16 @@ public void setDecommissioned(boolean isDecommissioned) {
}
@Override
+ public boolean isOverAllocationEnabled() {
+ return false;
+ }
+
+ @Override
+ public OverAllocationInfo getOverAllocationInfo() {
+ return null;
+ }
+
+ @Override
public NodeResourceMonitor getNodeResourceMonitor() {
return null;
}