diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 34a9829..509a6bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -1238,9 +1238,11 @@ protected synchronized void allocateContainersToNode(FiCaSchedulerNode node) { getCurrentAttemptForContainer(reservedContainer.getContainerId()); // Try to fulfill the reservation - LOG.info("Trying to fulfill reservation for application " - + reservedApplication.getApplicationId() + " on node: " - + node.getNodeID()); + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to fulfill reservation for application " + + reservedApplication.getApplicationId() + " on node: " + node + .getNodeID()); + } LeafQueue queue = ((LeafQueue) reservedApplication.getQueue()); assignment = @@ -1319,11 +1321,11 @@ protected synchronized void allocateContainersToNode(FiCaSchedulerNode node) { updateSchedulerHealth(lastNodeUpdateTime, node, assignment); } } else { - LOG.info("Skipping scheduling since node " - + node.getNodeID() - + " is reserved by application " - + node.getReservedContainer().getContainerId() - .getApplicationAttemptId()); + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping scheduling since node " + node.getNodeID() + + " is reserved by application " + node.getReservedContainer() + .getContainerId().getApplicationAttemptId()); + } } } @@ -1642,10 +1644,9 @@ public void recover(RMState state) throws Exception { @Override public void killReservedContainer(RMContainer container) { - if(LOG.isDebugEnabled()){ - LOG.debug(SchedulerEventType.KILL_RESERVED_CONTAINER + ":" - + container.toString()); - } + LOG.info("Preemption policy notifies to kill the reserved container:" + + container.toString()); + // To think: What happens if this is no longer a reserved container, for // e.g if the reservation became an allocation. super.completedContainer(container, @@ -1671,10 +1672,8 @@ public void markContainerForPreemption(ApplicationAttemptId aid, public synchronized void markContainerForKillable( RMContainer killableContainer) { - if (LOG.isDebugEnabled()) { - LOG.debug(SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE + ": container" - + killableContainer.toString()); - } + LOG.info("Preemption policy notifies the container:" + + killableContainer.toString() + " is killable."); if (!isLazyPreemptionEnabled) { super.completedContainer(killableContainer, SchedulerUtils diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index fbcb91c..da246d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import java.io.IOException; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -810,6 +811,9 @@ private void handleExcessReservedContainer(Resource clusterResource, CSAssignment assignment, FiCaSchedulerNode node, FiCaSchedulerApp app) { if (assignment.getExcessReservation() != null) { RMContainer excessReservedContainer = assignment.getExcessReservation(); + LOG.info( + "Release excessive reserved container: " + excessReservedContainer + .getContainerId()); if (excessReservedContainer.hasIncreaseReservation()) { unreserveIncreasedContainer(clusterResource, @@ -1329,6 +1333,7 @@ public void completedContainer(Resource clusterResource, synchronized (this) { Container container = rmContainer.getContainer(); + boolean reserved; // Inform the application & the node // Note: It's safe to assume that all state changes to RMContainer @@ -1337,22 +1342,26 @@ public void completedContainer(Resource clusterResource, if (rmContainer.getState() == RMContainerState.RESERVED) { removed = application.unreserve(rmContainer.getReservedPriority(), node, rmContainer); + reserved = true; } else { removed = application.containerCompleted(rmContainer, containerStatus, event, node.getPartition()); - + reserved = false; node.releaseContainer(container); } // Book-keeping if (removed) { - // Inform the ordering policy orderingPolicy.containerReleased(application, rmContainer); releaseResource(clusterResource, application, container.getResource(), node.getPartition(), rmContainer, false); + + LOG.info("Complete " + (reserved ? "reserved " : "") + "container=" + + rmContainer.getContainerId() + ", After release resource:" + + this); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 6fcd6c1..c653fc1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -643,10 +643,6 @@ private synchronized void internalReleaseResource(Resource clusterResource, releasedResource, node.getPartition(), changeResource); - if (LOG.isDebugEnabled()) { - LOG.debug("completedContainer " + this + ", cluster=" + clusterResource); - } - // Note that this is using an iterator on the childQueues so this can't // be called if already within an iterator for the childQueues. Like // from assignContainersToChildQueues. @@ -715,6 +711,9 @@ public void completedContainer(Resource clusterResource, rmContainer.getContainer().getResource(), false, completedChildQueue, sortQueues); + LOG.info("Complete container=" + rmContainer.getContainerId() + + ", After release resource:" + this); + // Inform the parent if (parent != null) { // complete my parent diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java index afac235..30652b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java @@ -73,10 +73,18 @@ protected CSAssignment getCSAssignmentFromAllocateResult( assignment.setType(result.getContainerNodeType()); if (result.getAllocationState() == AllocationState.RESERVED) { - // This is a reserved container - LOG.info("Reserved container " + " application=" - + application.getApplicationId() + " resource=" + allocatedResource - + " queue=" + this.toString() + " cluster=" + clusterResource); + if (rmContainer == null) { + // This is a reserved container + LOG.info("Reserved container " + " application=" + application + .getApplicationId() + " resource=" + allocatedResource + + " cluster=" + clusterResource); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Re-reserved container " + " application=" + application + .getApplicationId() + " resource=" + allocatedResource + + " cluster=" + clusterResource); + } + } assignment.getAssignmentInformation().addReservationDetails( updatedContainer.getId(), application.getCSLeafQueue().getQueuePath());