diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index d7ed7d1..2c107bd 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -825,25 +825,27 @@ private Resource assignContainer(
return capability;
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Resource request: " + capability + " exceeds the available"
+ + " resources of the node.");
+ }
+
// The desired container won't fit here, so reserve
if (isReservable(capability) && reserve(
pendingAsk.getPerAllocationResource(), node, reservedContainer, type,
schedulerKey)) {
- if (isWaitingForAMContainer()) {
- updateAMDiagnosticMsg(capability,
- " exceed the available resources of the node and the request is"
- + " reserved");
+ updateAMDiagnosticMsg(capability, " exceeds the available resources of "
+ + "the node and the request is reserved)");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(getName() + "'s resource request is reserved.");
}
return FairScheduler.CONTAINER_RESERVED;
} else {
- if (isWaitingForAMContainer()) {
- updateAMDiagnosticMsg(capability,
- " exceed the available resources of the node and the request cannot"
- + " be reserved");
- }
+ updateAMDiagnosticMsg(capability, " exceeds the available resources of "
+ + "the node and the request cannot be reserved)");
if (LOG.isDebugEnabled()) {
- LOG.debug("Couldn't creating reservation for " +
- getName() + ",at priority " + schedulerKey.getPriority());
+ LOG.debug("Couldn't creating reservation for app: " + getName()
+ + ", at priority " + schedulerKey.getPriority());
}
return Resources.none();
}
@@ -1020,10 +1022,9 @@ private boolean hasContainerForNode(SchedulerRequestKey key,
ret = false;
} else if (!getQueue().fitsInMaxShare(resource)) {
// The requested container must fit in queue maximum share
- if (isWaitingForAMContainer()) {
- updateAMDiagnosticMsg(resource,
- " exceeds current queue or its parents maximum resource allowed).");
- }
+ updateAMDiagnosticMsg(resource,
+ " exceeds current queue or its parents maximum resource allowed).");
+
ret = false;
}
@@ -1196,15 +1197,13 @@ public void updateDemand() {
@Override
public Resource assignContainer(FSSchedulerNode node) {
if (isOverAMShareLimit()) {
- if (isWaitingForAMContainer()) {
- PendingAsk amAsk = appSchedulingInfo.getNextPendingAsk();
- updateAMDiagnosticMsg(amAsk.getPerAllocationResource(),
- " exceeds maximum AM resource allowed).");
- }
-
+ PendingAsk amAsk = appSchedulingInfo.getNextPendingAsk();
+ updateAMDiagnosticMsg(amAsk.getPerAllocationResource(),
+ " exceeds maximum AM resource allowed).");
if (LOG.isDebugEnabled()) {
- LOG.debug("Skipping allocation because maxAMShare limit would " +
- "be exceeded");
+ LOG.debug("AM resource request: " + amAsk.getPerAllocationResource()
+ + " exceeds maximum AM resource allowed, "
+ + getQueue().dumpState());
}
return Resources.none();
}
@@ -1218,6 +1217,10 @@ public Resource assignContainer(FSSchedulerNode node) {
* @param reason the reason why AM doesn't get the resource
*/
private void updateAMDiagnosticMsg(Resource resource, String reason) {
+ if (!isWaitingForAMContainer()) {
+ return;
+ }
+
StringBuilder diagnosticMessageBldr = new StringBuilder();
diagnosticMessageBldr.append(" (Resource request: ");
diagnosticMessageBldr.append(resource);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 2754616..1854dc0 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -592,4 +592,25 @@ private boolean isStarvedForFairShare() {
boolean isStarved() {
return isStarvedForMinShare() || isStarvedForFairShare();
}
+
+ @Override
+ protected void dumpStateInternal(StringBuilder sb) {
+ sb.append("{Name: " + getName() +
+ ", Weight: " + weights +
+ ", Policy: " + policy.getName() +
+ ", FairShare: " + getFairShare() +
+ ", SteadyFairShare: " + getSteadyFairShare() +
+ ", MaxShare: " + maxShare +
+ ", MinShare: " + minShare +
+ ", ResourceUsage: " + getResourceUsage() +
+ ", Demand: " + getDemand() +
+ ", Runnable: " + getNumRunnableApps() +
+ ", NumPendingApps: " + getNumPendingApps() +
+ ", NonRunnable: " + getNumNonRunnableApps() +
+ ", MaxAMShare: " + maxAMShare +
+ ", MaxAMResource: " + computeMaxAMResource() +
+ ", AMResourceUsage: " + getAmResourceUsage() +
+ ", LastTimeAtMinShare: " + lastTimeAtMinShare +
+ "}");
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index 16570aa..41d161a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -320,4 +320,25 @@ public void recoverContainer(Resource clusterResource,
// TODO Auto-generated method stub
}
+
+ @Override
+ protected void dumpStateInternal(StringBuilder sb) {
+ sb.append("{Name: " + getName() +
+ ", Weight: " + weights +
+ ", Policy: " + policy.getName() +
+ ", FairShare: " + getFairShare() +
+ ", SteadyFairShare: " + getSteadyFairShare() +
+ ", MaxShare: " + maxShare +
+ ", MinShare: " + minShare +
+ ", ResourceUsage: " + getResourceUsage() +
+ ", Demand: " + getDemand() +
+ ", MaxAMShare: " + maxAMShare +
+ ", Runnable: " + getNumRunnableApps() +
+ "}");
+
+ for(FSQueue child : getChildQueues()) {
+ sb.append(", ");
+ child.dumpStateInternal(sb);
+ }
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index d87668d..d38e294 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -377,11 +377,22 @@ public abstract void collectSchedulerApplications(
* @return true if check passes (can assign) or false otherwise
*/
boolean assignContainerPreCheck(FSSchedulerNode node) {
- if (!Resources.fitsIn(getResourceUsage(), maxShare)
- || node.getReservedContainer() != null) {
+ if (node.getReservedContainer() != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Assigning container failed on node '" + node.getNodeName()
+ + " because it has reserved containers.");
+ }
+ return false;
+ } else if (!Resources.fitsIn(getResourceUsage(), maxShare)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Assigning container failed on node '" + node.getNodeName()
+ + " because queue resource usage is larger than MaxShare: "
+ + dumpState());
+ }
return false;
+ } else {
+ return true;
}
- return true;
}
/**
@@ -437,6 +448,11 @@ boolean fitsInMaxShare(Resource additionalResource) {
Resources.add(getResourceUsage(), additionalResource);
if (!Resources.fitsIn(usagePlusAddition, getMaxShare())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Resource usage plus resource request: " + usagePlusAddition
+ + " exceeds maximum resource allowed:" + getMaxShare()
+ + " in queue " + getName());
+ }
return false;
}
@@ -446,4 +462,23 @@ boolean fitsInMaxShare(Resource additionalResource) {
}
return true;
}
+
+ /**
+ * Recursively dump states of all queues.
+ *
+ * @return a string which holds all queue states
+ */
+ public String dumpState() {
+ StringBuilder sb = new StringBuilder();
+ dumpStateInternal(sb);
+ return sb.toString();
+ }
+
+
+ /**
+ * Recursively dump states of all queues.
+ *
+ * @param sb the {code StringBuilder} which holds queue states
+ */
+ protected abstract void dumpStateInternal(StringBuilder sb);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 97871e7..7716a57 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -343,6 +343,19 @@ public void run() {
}
/**
+ * Dump scheduler state including states of all queues.
+ */
+ private void dumpSchedulerState() {
+ FSQueue rootQueue = queueMgr.getRootQueue();
+ Resource clusterResource = getClusterResource();
+ LOG.debug("FairScheduler state: Cluster Capacity: " + clusterResource +
+ " Allocations: " + rootMetrics.getAllocatedResources() +
+ " Availability: " + Resource.newInstance(
+ rootMetrics.getAvailableMB(), rootMetrics.getAvailableVirtualCores()) +
+ rootQueue.dumpState());
+ }
+
+ /**
* Recompute the internal variables used by the scheduler - per-job weights,
* fair shares, deficits, minimum slot allocations, and amount of used and
* required resources per job.
@@ -366,12 +379,7 @@ protected void update() {
if (LOG.isDebugEnabled()) {
if (--updatesToSkipForDebug < 0) {
updatesToSkipForDebug = UPDATE_DEBUG_FREQUENCY;
- LOG.debug("Cluster Capacity: " + clusterResource +
- " Allocations: " + rootMetrics.getAllocatedResources() +
- " Availability: " + Resource.newInstance(
- rootMetrics.getAvailableMB(),
- rootMetrics.getAvailableVirtualCores()) +
- " Demand: " + rootQueue.getDemand());
+ dumpSchedulerState();
}
long duration = getClock().getTime() - start;
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index b1e412b..cccfd8f 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -5096,4 +5096,74 @@ public void testUpdateDemand() throws IOException {
Resources.equals(bQueue.getDemand(), maxResource));
}
+ @Test
+ public void testDumpState() throws IOException {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("");
+ out.println("");
+ out.println("");
+ out.println(" ");
+ out.println(" 1");
+ out.println(" ");
+ out.println("");
+ out.println("");
+ out.close();
+
+ ControlledClock clock = new ControlledClock();
+ scheduler.setClock(clock);
+
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ FSLeafQueue child1 =
+ scheduler.getQueueManager().getLeafQueue("parent.child1", false);
+ Resource resource = Resource.newInstance(4 * GB, 4);
+ child1.setMaxShare(resource);
+ FSAppAttempt app = mock(FSAppAttempt.class);
+ Mockito.when(app.getDemand()).thenReturn(resource);
+ Mockito.when(app.getResourceUsage()).thenReturn(resource);
+ child1.addAppSchedulable(app);
+ child1.updateDemand();
+
+ String childQueueString = "{Name: root.parent.child1,"
+ + " Weight: ,"
+ + " Policy: fair,"
+ + " FairShare: ,"
+ + " SteadyFairShare: ,"
+ + " MaxShare: ,"
+ + " MinShare: ,"
+ + " ResourceUsage: ,"
+ + " Demand: ,"
+ + " Runnable: 1,"
+ + " NumPendingApps: 0,"
+ + " NonRunnable: 0,"
+ + " MaxAMShare: 0.5,"
+ + " MaxAMResource: ,"
+ + " AMResourceUsage: ,"
+ + " LastTimeAtMinShare: " + clock.getTime()
+ + "}";
+
+ assertTrue(child1.dumpState().equals(childQueueString));
+ FSParentQueue parent =
+ scheduler.getQueueManager().getParentQueue("parent", false);
+ parent.setMaxShare(resource);
+
+ String parentQueueString = "{Name: root.parent,"
+ + " Weight: ,"
+ + " Policy: fair,"
+ + " FairShare: ,"
+ + " SteadyFairShare: ,"
+ + " MaxShare: ,"
+ + " MinShare: ,"
+ + " ResourceUsage: ,"
+ + " Demand: ,"
+ + " MaxAMShare: 0.5,"
+ + " Runnable: 0}";
+
+ assertTrue(parent.dumpState().equals(
+ parentQueueString + ", " + childQueueString));
+ }
}