diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java
index c3f8595..e15f5b2 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java
@@ -38,14 +38,36 @@
@Private
@Unstable
public static NodeId newInstance(String host, int port) {
+ return newInstance(host, port, false);
+ }
+
+ @Private
+ @Unstable
+ public static NodeId newInstance(String host, int port, boolean usePortForName) {
NodeId nodeId = Records.newRecord(NodeId.class);
nodeId.setHost(host);
nodeId.setPort(port);
+ nodeId.setNodeName((usePortForName) ? host + ":" + port : host);
nodeId.build();
return nodeId;
}
/**
+ * Get the name of the node, the name is used by the scheduler to
+ * perform allocations. Typically, the name is the hostname of the node.
+ * If the NodeManager is configure to use the port, the name of the node
+ * is the hostname:port.
+ * @return name of the node
+ */
+ @Public
+ @Stable
+ public abstract String getNodeName();
+
+ @Private
+ @Unstable
+ protected abstract void setNodeName(String nodeName);
+
+ /**
* Get the hostname of the node.
* @return hostname of the node
*/
@@ -71,9 +93,12 @@ public static NodeId newInstance(String host, int port) {
@Override
public String toString() {
- return this.getHost() + ":" + this.getPort();
+ return this.getHost() + ":" + this.getPort() + " name: " +
+ this.getNodeName();
}
+ //we don't need to use nodeName because nodeName is by definition
+ //either host or host:port
@Override
public int hashCode() {
final int prime = 493217;
@@ -83,6 +108,8 @@ public int hashCode() {
return result;
}
+ //we don't need to compare nodeName because nodeName is by definition
+ //either host or host:port
@Override
public boolean equals(Object obj) {
if (this == obj)
@@ -99,6 +126,8 @@ public boolean equals(Object obj) {
return true;
}
+ //we don't need to compare nodeName because nodeName is by definition
+ //either host or host:port
@Override
public int compareTo(NodeId other) {
int hostCompare = this.getHost().compareTo(other.getHost());
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index d7aa413..59630d8 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -141,13 +141,25 @@ public static boolean isAnyLocation(String hostName) {
public abstract void setPriority(Priority priority);
/**
- * Get the resource (e.g. host/rack) on which the allocation
+ * Get the resource (e.g. nodeName/rack) on which the allocation
* is desired.
- *
+ *
+ * The nodeName typically is the hostname of the name.
+ *
+ * ApplicationMasters should not care about this distinction in regular setups
+ * as there is a single NodeManager per node and the node name matches the
+ * hostname.
+ *
+ * NodeManagers can be configured to use the port also for their names,
+ * producing a hostname:port name. The later is useful for testing
+ * and when for some reason there is more than one NodeManager per node.
+ * If the port is being used for the node name, ApplicationMasters can
+ * determine the name of a node from the NodeReport.
+ *
* A special value of * signifies that any resource
- * (host/rack) is acceptable.
+ * (nodeName/rack) is acceptable.
*
- * @return resource (e.g. host/rack) on which the allocation
+ * @return resource (e.g. nodeName/rack) on which the allocation
* is desired
*/
@Public
@@ -155,13 +167,25 @@ public static boolean isAnyLocation(String hostName) {
public abstract String getResourceName();
/**
- * Set the resource name (e.g. host/rack) on which the allocation
+ * Set the resource name (e.g. nodeName/rack) on which the allocation
* is desired.
- *
+ *
+ * The nodeName typically is the hostname of the name.
+ *
+ * ApplicationMasters should not care about this distinction in regular setups
+ * as there is a single NodeManager per node and the node name matches the
+ * hostname.
+ * *
+ * NodeManagers can be configured to use the port also for their names,
+ * producing a hostname:port name. The later is useful for testing
+ * and when for some reason there is more than one NodeManager per node.
+ * If the port is being used for the node name, ApplicationMasters can
+ * determine the name of a node from the NodeReport.
+ *
* A special value of * signifies that any resource name
- * (e.g. host/rack) is acceptable.
+ * (e.g. nodeName/rack) is acceptable.
*
- * @param resourceName (e.g. host/rack) on which the
+ * @param resourceName (e.g. nodeName/rack) on which the
* allocation is desired
*/
@Public
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 065fb63..bd46392 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -332,6 +332,11 @@
public static final int DEFAULT_NM_PORT = 0;
public static final String DEFAULT_NM_ADDRESS = "0.0.0.0:"
+ DEFAULT_NM_PORT;
+
+ /** If the node name used for scheduling matching shoudl contain the port.*/
+ public static final String NM_USE_PORT_FOR_NAME = NM_PREFIX +
+ ".use.port.for.name";
+ public static final boolean DEFAULT_NM_USE_PORT_FOR_NAME = false;
/** who will execute(launch) the containers.*/
public static final String NM_CONTAINER_EXECUTOR =
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index a27cbb7..bec2f05 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -164,6 +164,7 @@ enum NodeStateProto {
message NodeIdProto {
optional string host = 1;
optional int32 port = 2;
+ optional string nodeName = 3;
}
message NodeReportProto {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeIdPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeIdPBImpl.java
index 4c725fa..3f89ed7 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeIdPBImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeIdPBImpl.java
@@ -45,6 +45,18 @@ public NodeIdProto getProto() {
}
@Override
+ public String getNodeName() {
+ Preconditions.checkNotNull(proto);
+ return proto.getNodeName();
+ }
+
+ @Override
+ protected void setNodeName(String nodeName) {
+ Preconditions.checkNotNull(builder);
+ builder.setNodeName(nodeName);
+ }
+
+ @Override
public String getHost() {
Preconditions.checkNotNull(proto);
return proto.getHost();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index f8a5ea2..ea9ab86 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -263,10 +263,18 @@ protected void serviceStart() throws Exception {
" server is still starting.");
this.setBlockNewContainerRequests(true);
server.start();
+ boolean usePortForName = getConfig().getBoolean(
+ YarnConfiguration.NM_USE_PORT_FOR_NAME,
+ YarnConfiguration.DEFAULT_NM_USE_PORT_FOR_NAME);
InetSocketAddress connectAddress = NetUtils.getConnectAddress(server);
NodeId nodeId = NodeId.newInstance(
connectAddress.getAddress().getCanonicalHostName(),
- connectAddress.getPort());
+ connectAddress.getPort(), usePortForName);
+ if (usePortForName) {
+ LOG.info("Using port for node name, the name of the node is: " +
+ nodeId.getNodeName());
+ }
+
((NodeManager.NMContext)context).setNodeId(nodeId);
this.context.getNMTokenSecretManager().setNodeId(nodeId);
this.context.getContainerTokenSecretManager().setNodeId(nodeId);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 1ff00be..6f8144d 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -281,7 +281,7 @@ synchronized private void allocateNodeLocal(
// Update future requirements
nodeLocalRequest.setNumContainers(nodeLocalRequest.getNumContainers() - 1);
if (nodeLocalRequest.getNumContainers() == 0) {
- this.requests.get(priority).remove(node.getHostName());
+ this.requests.get(priority).remove(node.getNodeName());
}
ResourceRequest rackLocalRequest = requests.get(priority).get(
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index a08ba70..d3a6a97 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -30,10 +30,10 @@
public abstract class SchedulerNode {
/**
- * Get hostname.
- * @return hostname
+ * Get node name.
+ * @return node name
*/
- public abstract String getHostName();
+ public abstract String getNodeName();
/**
* Get rackname.
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index c2c5d27..41b3f5e 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -801,7 +801,7 @@ private synchronized FiCaSchedulerApp getApplication(
assignContainers(Resource clusterResource, FiCaSchedulerNode node) {
if(LOG.isDebugEnabled()) {
- LOG.debug("assignContainers: node=" + node.getHostName()
+ LOG.debug("assignContainers: node=" + node.getNodeName()
+ " #applications=" + activeApplications.size());
}
@@ -1130,7 +1130,7 @@ private CSAssignment assignContainersOnNode(Resource clusterResource,
// Data-local
ResourceRequest nodeLocalResourceRequest =
- application.getResourceRequest(priority, node.getHostName());
+ application.getResourceRequest(priority, node.getNodeName());
if (nodeLocalResourceRequest != null) {
assigned =
assignNodeLocalContainers(clusterResource, nodeLocalResourceRequest,
@@ -1257,7 +1257,7 @@ boolean canAssign(FiCaSchedulerApp application, Priority priority,
if (type == NodeType.NODE_LOCAL) {
// Now check if we need containers on this host...
ResourceRequest nodeLocalRequest =
- application.getResourceRequest(priority, node.getHostName());
+ application.getResourceRequest(priority, node.getNodeName());
if (nodeLocalRequest != null) {
return nodeLocalRequest.getNumContainers() > 0;
}
@@ -1302,7 +1302,7 @@ private Resource assignContainer(Resource clusterResource, FiCaSchedulerNode nod
FiCaSchedulerApp application, Priority priority,
ResourceRequest request, NodeType type, RMContainer rmContainer) {
if (LOG.isDebugEnabled()) {
- LOG.debug("assignContainers: node=" + node.getHostName()
+ LOG.debug("assignContainers: node=" + node.getNodeName()
+ " application=" + application.getApplicationId().getId()
+ " priority=" + priority.getPriority()
+ " request=" + request + " type=" + type);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
index bb9ba92..b3d6ff8 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
@@ -59,11 +59,13 @@
new HashMap();
private final RMNode rmNode;
+ private final String nodeName;
public FiCaSchedulerNode(RMNode node) {
this.rmNode = node;
this.availableResource.setMemory(node.getTotalCapability().getMemory());
this.availableResource.setVirtualCores(node.getTotalCapability().getVirtualCores());
+ this.nodeName = node.getNodeID().getNodeName();
}
public RMNode getRMNode() {
@@ -79,8 +81,8 @@ public String getHttpAddress() {
}
@Override
- public String getHostName() {
- return this.rmNode.getHostName();
+ public String getNodeName() {
+ return this.nodeName;
}
@Override
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerUtils.java
index 1e96949..9bece9b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerUtils.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerUtils.java
@@ -24,9 +24,9 @@
public static boolean isBlacklisted(FiCaSchedulerApp application,
FiCaSchedulerNode node, Log LOG) {
- if (application.isBlacklisted(node.getHostName())) {
+ if (application.isBlacklisted(node.getNodeName())) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Skipping 'host' " + node.getHostName() +
+ LOG.debug("Skipping 'host' " + node.getNodeName() +
" for " + application.getApplicationId() +
" since it has been blacklisted");
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
index bb3190b..14ec99c 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
@@ -185,7 +185,7 @@ public Container createContainer(
*/
private void reserve(Priority priority, FSSchedulerNode node,
Container container, boolean alreadyReserved) {
- LOG.info("Making reservation: node=" + node.getHostName() +
+ LOG.info("Making reservation: node=" + node.getNodeName() +
" app_id=" + app.getApplicationId());
if (!alreadyReserved) {
getMetrics().reserveResource(app.getUser(), container.getResource());
@@ -309,7 +309,7 @@ private Resource assignContainer(FSSchedulerNode node, boolean reserved) {
ResourceRequest rackLocalRequest = app.getResourceRequest(priority,
node.getRackName());
ResourceRequest localRequest = app.getResourceRequest(priority,
- node.getHostName());
+ node.getNodeName());
if (localRequest != null && !localRequest.getRelaxLocality()) {
LOG.warn("Relax locality off is not supported on local request: "
@@ -369,7 +369,7 @@ public Resource assignContainer(FSSchedulerNode node) {
public boolean hasContainerForNode(Priority prio, FSSchedulerNode node) {
ResourceRequest anyRequest = app.getResourceRequest(prio, ResourceRequest.ANY);
ResourceRequest rackRequest = app.getResourceRequest(prio, node.getRackName());
- ResourceRequest nodeRequest = app.getResourceRequest(prio, node.getHostName());
+ ResourceRequest nodeRequest = app.getResourceRequest(prio, node.getNodeName());
return
// There must be outstanding requests at the given priority:
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
index cc15a5d..6a91c30 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
@@ -63,10 +63,12 @@
new HashMap();
private final RMNode rmNode;
-
+ private final String nodeName;
+
public FSSchedulerNode(RMNode node) {
this.rmNode = node;
this.availableResource = Resources.clone(node.getTotalCapability());
+ this.nodeName = node.getNodeID().getNodeName();
}
public RMNode getRMNode() {
@@ -82,8 +84,8 @@ public String getHttpAddress() {
}
@Override
- public String getHostName() {
- return rmNode.getHostName();
+ public String getNodeName() {
+ return nodeName;
}
@Override
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 16b543c..050be49 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -881,7 +881,7 @@ private synchronized void nodeUpdate(RMNode nm) {
if (LOG.isDebugEnabled()) {
LOG.debug("nodeUpdate: " + nm + " cluster capacity: " + clusterCapacity);
}
- eventLog.log("HEARTBEAT", nm.getHostName());
+ eventLog.log("HEARTBEAT", nm.getNodeID().getNodeName());
FSSchedulerNode node = nodes.get(nm.getNodeID());
List containerInfoList = nm.pullContainerUpdates();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index d971f3b..d665540 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -490,7 +490,7 @@ private int assignNodeLocalContainers(FiCaSchedulerNode node,
FiCaSchedulerApp application, Priority priority) {
int assignedContainers = 0;
ResourceRequest request =
- application.getResourceRequest(priority, node.getHostName());
+ application.getResourceRequest(priority, node.getNodeName());
if (request != null) {
// Don't allocate on this node if we don't need containers on this rack
ResourceRequest rackRequest =
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 83b81a1..f1b2f6c 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -200,19 +200,20 @@ public long getLastHealthReportTime() {
};
private static RMNode buildRMNode(int rack, final Resource perNode, NodeState state, String httpAddr) {
- return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null);
+ return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, false);
}
private static RMNode buildRMNode(int rack, final Resource perNode,
- NodeState state, String httpAddr, int hostnum, String hostName) {
+ NodeState state, String httpAddr, int hostnum, String hostName, boolean
+ usePortForName) {
final String rackName = "rack"+ rack;
final int nid = hostnum;
final String nodeAddr = hostName + ":" + nid;
- final int port = 123;
+ final int port = hostnum;
if (hostName == null) {
hostName = "host"+ nid;
}
- final NodeId nodeID = NodeId.newInstance(hostName, port);
+ final NodeId nodeID = NodeId.newInstance(hostName, port, usePortForName);
final String httpAddress = httpAddr;
String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe";
@@ -230,12 +231,19 @@ public static RMNode newNodeInfo(int rack, final Resource perNode) {
}
public static RMNode newNodeInfo(int rack, final Resource perNode, int hostnum) {
- return buildRMNode(rack, perNode, null, "localhost:0", hostnum, null);
+ return buildRMNode(rack, perNode, null, "localhost:0", hostnum, null, false);
}
public static RMNode newNodeInfo(int rack, final Resource perNode,
int hostnum, String hostName) {
- return buildRMNode(rack, perNode, null, "localhost:0", hostnum, hostName);
+ return buildRMNode(rack, perNode, null, "localhost:0", hostnum, hostName,
+ false);
+ }
+
+ public static RMNode newNodeInfo(int rack, final Resource perNode,
+ int hostnum, String hostName, boolean usePortForName) {
+ return buildRMNode(rack, perNode, null, "localhost:0", hostnum, hostName,
+ usePortForName);
}
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index 014385c..3c55b42 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -26,7 +26,6 @@
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.util.HashMap;
@@ -126,7 +125,7 @@ public CSAssignment answer(InvocationOnMock invocation) throws Throwable {
throw new Exception();
} catch (Exception e) {
LOG.info("FOOBAR q.assignContainers q=" + queue.getQueueName() +
- " alloc=" + allocation + " node=" + node.getHostName());
+ " alloc=" + allocation + " node=" + node.getNodeName());
}
final Resource allocatedResource = Resources.createResource(allocation);
if (queue instanceof ParentQueue) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index c5dbfde..0348081 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -138,7 +138,7 @@ public CSAssignment answer(InvocationOnMock invocation) throws Throwable {
throw new Exception();
} catch (Exception e) {
LOG.info("FOOBAR q.assignContainers q=" + queue.getQueueName() +
- " alloc=" + allocation + " node=" + node.getHostName());
+ " alloc=" + allocation + " node=" + node.getNodeName());
}
final Resource allocatedResource = Resources.createResource(allocation);
if (queue instanceof ParentQueue) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 8be344c..6e2207d 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -2009,6 +2009,49 @@ public void testNoMoreCpuOnNode() {
assertEquals(1, app.getLiveContainers().size());
}
+ @Test
+ public void testHostPortNodeName() {
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1,
+ "127.0.0.1", true);
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 2,
+ "127.0.0.1", true);
+ NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+ scheduler.handle(nodeEvent2);
+
+ ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1",
+ "user1", 0);
+
+ ResourceRequest nodeRequest = createResourceRequest(1024,
+ node1.getNodeID().getNodeName(), 1, 1, true);
+ ResourceRequest rackRequest = createResourceRequest(1024,
+ node1.getRackName(), 1, 1, false);
+ ResourceRequest anyRequest = createResourceRequest(1024, ResourceRequest.ANY,
+ 1, 1, false);
+ createSchedulingRequestExistingApplication(nodeRequest, attId1);
+ createSchedulingRequestExistingApplication(rackRequest, attId1);
+ createSchedulingRequestExistingApplication(anyRequest, attId1);
+
+ scheduler.update();
+
+ NodeUpdateSchedulerEvent node1UpdateEvent = new NodeUpdateSchedulerEvent(node1);
+ NodeUpdateSchedulerEvent node2UpdateEvent = new NodeUpdateSchedulerEvent(node2);
+
+ // no matter how many heartbeats, node2 should never get a container
+ FSSchedulerApp app = scheduler.applications.get(attId1);
+ for (int i = 0; i < 10; i++) {
+ scheduler.handle(node2UpdateEvent);
+ assertEquals(0, app.getLiveContainers().size());
+ assertEquals(0, app.getReservedContainers().size());
+ }
+ // then node1 should get the container
+ scheduler.handle(node1UpdateEvent);
+ assertEquals(1, app.getLiveContainers().size());
+ }
+
+ @Test
public void testBasicDRFAssignment() throws Exception {
RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(8192, 5));
NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNCluster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNCluster.java
new file mode 100644
index 0000000..2723f1e
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNCluster.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server;
+
+import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Test;
+
+public class TestMiniYARNCluster {
+
+ @Test
+ public void testHostForNodeName() throws Exception {
+ testName(false);
+ }
+
+ @Test
+ public void testHostPortForNodeName() throws Exception {
+ testName(true);
+ }
+
+ public void testName(boolean usePort) throws Exception {
+ MiniYARNCluster miniCluster = new MiniYARNCluster("test", 2, 1, 1);
+ try {
+ Configuration conf = new YarnConfiguration();
+ // if false we are testing that the default is false
+ if (usePort) {
+ conf.setBoolean(YarnConfiguration.NM_USE_PORT_FOR_NAME, usePort);
+ }
+ miniCluster.init(conf);
+ miniCluster.start();
+ for (int i = 0; i < 2; i++) {
+ NodeId nId = miniCluster.getNodeManager(0).getNMContext().getNodeId();
+ if (usePort) {
+ Assert.assertEquals(nId.getHost() + ":" + nId.getPort(),
+ nId.getNodeName());
+ } else {
+ Assert.assertEquals(nId.getHost(), nId.getNodeName());
+ }
+ }
+ } finally {
+ miniCluster.stop();
+ }
+ }
+}