diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 45d5bba..029c7fe 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -249,12 +249,30 @@ public synchronized NodeType getAllowedLocalityLevel(Priority priority, rackLocalityThreshold; // Relax locality constraints once we've surpassed threshold. - if (getSchedulingOpportunities(priority) > (numNodes * threshold)) { + int schedulingOpportunities = getSchedulingOpportunities(priority); + double thresholdNum = numNodes * threshold; + if (schedulingOpportunities > thresholdNum) { if (allowed.equals(NodeType.NODE_LOCAL)) { + if (LOG.isDebugEnabled()) { + LOG.debug("SchedulingOpportunities count is " + + schedulingOpportunities + + ", more than nodeLocalityThreshold num " + thresholdNum + + ", change allowedLocality from NODE_LOCAL to " + + NodeType.RACK_LOCAL + ", priority:" + priority + + ", app attempt id:" + this.attemptId); + } allowedLocalityLevel.put(priority, NodeType.RACK_LOCAL); resetSchedulingOpportunities(priority); } else if (allowed.equals(NodeType.RACK_LOCAL)) { + if (LOG.isDebugEnabled()) { + LOG.debug("SchedulingOpportunities count is " + + schedulingOpportunities + + ", more than rackLocalityThreshold num " + thresholdNum + + ", change allowedLocality from RACK_LOCAL to " + + NodeType.OFF_SWITCH + ", priority:" + priority + + ", app attempt id:" + this.attemptId); + } allowedLocalityLevel.put(priority, NodeType.OFF_SWITCH); resetSchedulingOpportunities(priority); } @@ -302,9 +320,23 @@ public synchronized NodeType getAllowedLocalityLevelByTime(Priority priority, if (waitTime > thresholdTime) { if (allowed.equals(NodeType.NODE_LOCAL)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting time is " + waitTime + + " ms, more than nodeLocalityDelay time " + nodeLocalityDelayMs + + " ms" + ", change allowedLocality from NODE_LOCAL to " + + NodeType.RACK_LOCAL + ", priority:" + priority + + ", app attempt id:" + this.attemptId); + } allowedLocalityLevel.put(priority, NodeType.RACK_LOCAL); resetSchedulingOpportunities(priority, currentTimeMs); } else if (allowed.equals(NodeType.RACK_LOCAL)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting time is " + waitTime + + " ms, more than rackLocalityDelay time " + rackLocalityDelayMs + + " ms" + ", change allowedLocality from RACK_LOCAL to " + + NodeType.OFF_SWITCH + ", priority:" + priority + + ", app attempt id:" + this.attemptId); + } allowedLocalityLevel.put(priority, NodeType.OFF_SWITCH); resetSchedulingOpportunities(priority, currentTimeMs); } @@ -606,6 +638,14 @@ private Resource assignContainer(FSSchedulerNode node, boolean reserved) { if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0 && localRequest != null && localRequest.getNumContainers() != 0) { + if (LOG.isDebugEnabled()) { + LOG.debug("Assign container on " + node.getNodeName() + + " node, requestType:" + NodeType.NODE_LOCAL + + ", allowedLocality:" + allowedLocality + + ", schedulingOpportunities:" + + this.getSchedulingOpportunities(priority) + ", priority:" + + priority + ", app attempt id:" + this.attemptId); + } return assignContainer(node, localRequest, NodeType.NODE_LOCAL, reserved); } @@ -617,6 +657,14 @@ private Resource assignContainer(FSSchedulerNode node, boolean reserved) { if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0 && (allowedLocality.equals(NodeType.RACK_LOCAL) || allowedLocality.equals(NodeType.OFF_SWITCH))) { + if (LOG.isDebugEnabled()) { + LOG.debug("Assign container on " + node.getNodeName() + + " node, requestType:" + NodeType.RACK_LOCAL + + ", allowedLocality:" + allowedLocality + + ", schedulingOpportunities:" + + this.getSchedulingOpportunities(priority) + ", priority:" + + priority + ", app attempt id:" + this.attemptId); + } return assignContainer(node, rackLocalRequest, NodeType.RACK_LOCAL, reserved); } @@ -631,10 +679,26 @@ private Resource assignContainer(FSSchedulerNode node, boolean reserved) { offSwitchRequest.getNumContainers() != 0) { if (!hasNodeOrRackLocalRequests(priority) || allowedLocality.equals(NodeType.OFF_SWITCH)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Assign container on " + node.getNodeName() + + " node, requestType:" + NodeType.OFF_SWITCH + + ", allowedLocality:" + allowedLocality + + ", schedulingOpportunities:" + + this.getSchedulingOpportunities(priority) + ", priority:" + + priority + ", app attempt id:" + this.attemptId); + } return assignContainer( node, offSwitchRequest, NodeType.OFF_SWITCH, reserved); } } + + if (LOG.isDebugEnabled()) { + LOG.debug("Can't assign container on " + node.getNodeName() + + " node, allowedLocality:" + allowedLocality + + ", schedulingOpportunities:" + + this.getSchedulingOpportunities(priority) + ", priority:" + + priority + ", app attempt id:" + this.attemptId); + } } } return Resources.none();