diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index 316a450..a651ec5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -379,7 +379,10 @@ public void setAMContainer(boolean isAMContainer) { @Override public void handle(RMContainerEvent event) { - LOG.debug("Processing " + event.getContainerId() + " of type " + event.getType()); + if (LOG.isDebugEnabled()) { + LOG.debug("Processing " + event.getContainerId() + " of type " + event + .getType()); + } try { writeLock.lock(); RMContainerState oldState = getState(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java index f03663a..a5cfd88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java @@ -151,11 +151,13 @@ public synchronized void allocateContainer(RMContainer rmContainer) { launchedContainers.put(container.getId(), rmContainer); - LOG.info("Assigned container " + container.getId() + " of capacity " - + container.getResource() + " on host " + rmNode.getNodeAddress() - + ", which has " + numContainers + " containers, " - + getUsedResource() + " used and " + getAvailableResource() - + " available after allocation"); + if (LOG.isDebugEnabled()) { + LOG.debug("Assigned container " + container.getId() + " of capacity " + + container.getResource() + " on host " + rmNode.getNodeAddress() + + ", which has " + numContainers + " containers, " + + getUsedResource() + " used and " + getAvailableResource() + + " available after allocation"); + } } /** @@ -214,11 +216,13 @@ public synchronized void releaseContainer(Container container) { updateResource(container); } - LOG.info("Released container " + container.getId() + " of capacity " - + container.getResource() + " on host " + rmNode.getNodeAddress() - + ", which currently has " + numContainers + " containers, " - + getUsedResource() + " used and " + getAvailableResource() - + " available" + ", release resources=" + true); + if (LOG.isDebugEnabled()) { + LOG.debug("Released container " + container.getId() + " of capacity " + + container.getResource() + " on host " + rmNode.getNodeAddress() + + ", which currently has " + numContainers + " containers, " + + getUsedResource() + " used and " + getAvailableResource() + + " available" + ", release resources=" + true); + } } private synchronized void addAvailableResource(Resource resource) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 6287deb..946bd10 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -52,6 +52,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; @@ -124,14 +126,16 @@ synchronized public void containerCompleted(RMContainer rmContainer, // Inform the container rmContainer.handle( - new RMContainerFinishedEvent( - containerId, - containerStatus, - event) - ); - LOG.info("Completed container: " + rmContainer.getContainerId() + - " in state: " + rmContainer.getState() + " event:" + event); - + new RMContainerFinishedEvent( + containerId, + containerStatus, + event) + ); + if (LOG.isDebugEnabled()) { + LOG.debug("Completed container: " + rmContainer.getContainerId() + + " in state: " + rmContainer.getState() + " event:" + event); + } + // Remove from the list of containers liveContainers.remove(rmContainer.getContainerId()); @@ -143,6 +147,7 @@ synchronized public void containerCompleted(RMContainer rmContainer, Resource containerResource = rmContainer.getContainer().getResource(); queue.getMetrics().releaseResources(getUser(), 1, containerResource); this.attemptResourceUsage.decUsed(containerResource); + ((FSQueue)queue).decrementResourceUsage(container.getResource()); // remove from preemption map if it is completed preemptionMap.remove(rmContainer); @@ -349,6 +354,7 @@ else if (allowed.equals(NodeType.RACK_LOCAL) && List resourceRequestList = appSchedulingInfo.allocate( type, node, priority, request, container); this.attemptResourceUsage.incUsed(container.getResource()); + ((FSQueue)queue).incrementResourceUsage(container.getResource()); // Update resource requests related to "request" and store in RMContainer ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList); @@ -370,6 +376,17 @@ else if (allowed.equals(NodeType.RACK_LOCAL) && return rmContainer; } + @Override + public synchronized void recoverContainer(SchedulerNode node, + RMContainer rmContainer) { + super.recoverContainer(node, rmContainer); + if (!rmContainer.getState().equals(org.apache.hadoop.yarn.server + .resourcemanager.rmcontainer.RMContainerState.COMPLETED)) { + ((FSQueue)queue).incrementResourceUsage(rmContainer.getContainer() + .getResource()); + } + } + /** * Should be called when the scheduler assigns a container at a higher * degree of locality than the current threshold. Reset the allowed locality @@ -388,6 +405,8 @@ public void addPreemption(RMContainer container, long time) { assert preemptionMap.get(container) == null; preemptionMap.put(container, time); Resources.addTo(preemptedResources, container.getAllocatedResource()); + // Preempted resources should bring down resource usage. + ((FSQueue)queue).decrementResourceUsage(container.getAllocatedResource()); } public Long getContainerPreemptionTime(RMContainer container) { @@ -408,10 +427,12 @@ public Resource getPreemptedResources() { } public void resetPreemptedResources() { + ((FSQueue)queue).incrementResourceUsage(preemptedResources); preemptedResources = Resources.createResource(0); for (RMContainer container : getPreemptionContainers()) { Resources.addTo(preemptedResources, container.getAllocatedResource()); } + ((FSQueue)queue).decrementResourceUsage(preemptedResources); } public void clearPreemptedResources() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index 04dbd2f..2ed83ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -51,9 +51,10 @@ FSLeafQueue.class.getName()); private final List runnableApps = // apps that are runnable - new ArrayList(); + new ArrayList(); private final List nonRunnableApps = - new ArrayList(); + new ArrayList(); + // get a lock with fair distribution for app list updates private final ReadWriteLock rwl = new ReentrantReadWriteLock(true); private final Lock readLock = rwl.readLock(); @@ -237,23 +238,6 @@ public Resource getDemand() { return demand; } - @Override - public Resource getResourceUsage() { - Resource usage = Resources.createResource(0); - readLock.lock(); - try { - for (FSAppAttempt app : runnableApps) { - Resources.addTo(usage, app.getResourceUsage()); - } - for (FSAppAttempt app : nonRunnableApps) { - Resources.addTo(usage, app.getResourceUsage()); - } - } finally { - readLock.unlock(); - } - return usage; - } - public Resource getAmResourceUsage() { return amResourceUsage; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java index f74106a..a205107 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java @@ -92,15 +92,6 @@ public Resource getDemand() { } @Override - public Resource getResourceUsage() { - Resource usage = Resources.createResource(0); - for (FSQueue child : childQueues) { - Resources.addTo(usage, child.getResourceUsage()); - } - return usage; - } - - @Override public void updateDemand() { // Compute demand by iterating through apps in the queue // Limit demand to maxResources diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index ade2880..6124e9e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -45,7 +45,7 @@ @Unstable public abstract class FSQueue implements Queue, Schedulable { private static final Log LOG = LogFactory.getLog( - FSQueue.class.getName()); + FSQueue.class.getName()); private Resource fairShare = Resources.createResource(0, 0); private Resource steadyFairShare = Resources.createResource(0, 0); @@ -62,6 +62,7 @@ private long fairSharePreemptionTimeout = Long.MAX_VALUE; private long minSharePreemptionTimeout = Long.MAX_VALUE; private float fairSharePreemptionThreshold = 0.5f; + private final Resource usage = Resource.newInstance(0, 0); public FSQueue(String name, FairScheduler scheduler, FSParentQueue parent) { this.name = name; @@ -71,7 +72,26 @@ public FSQueue(String name, FairScheduler scheduler, FSParentQueue parent) { metrics.setMaxShare(getMaxShare()); this.parent = parent; } - + + @Override + public Resource getResourceUsage() { + return usage; + } + + public synchronized void incrementResourceUsage(Resource res) { + Resources.addTo(this.usage, res); + if (parent != null) { + parent.incrementResourceUsage(res); + } + } + + public synchronized void decrementResourceUsage(Resource res) { + Resources.subtractFrom(this.usage, res); + if (parent != null) { + parent.decrementResourceUsage(res); + } + } + public String getName() { return name; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index f481de5..92cf089 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -454,9 +454,11 @@ protected void warnOrKillContainer(RMContainer container) { // TODO: Not sure if this ever actually adds this to the list of cleanup // containers on the RMNode (see SchedulerNode.releaseContainer()). completedContainer(container, status, RMContainerEventType.KILL); - LOG.info("Killing container" + container + - " (after waiting for premption for " + - (getClock().getTime() - time) + "ms)"); + if (LOG.isDebugEnabled()) { + LOG.debug("Killing container" + container + + " (after waiting for premption for " + + (getClock().getTime() - time) + "ms)"); + } } } else { // track the request in the FSAppAttempt itself @@ -676,8 +678,8 @@ protected synchronized void addApplicationAttempt( } } else { rmContext.getDispatcher().getEventHandler().handle( - new RMAppAttemptEvent(applicationAttemptId, - RMAppAttemptEventType.ATTEMPT_ADDED)); + new RMAppAttemptEvent(applicationAttemptId, + RMAppAttemptEventType.ATTEMPT_ADDED)); } } @@ -826,9 +828,11 @@ protected synchronized void completedContainer(RMContainer rmContainer, updateRootQueueMetrics(); } - LOG.info("Application attempt " + application.getApplicationAttemptId() - + " released container " + container.getId() + " on node: " + node - + " with event: " + event); + if (LOG.isDebugEnabled()) { + LOG.debug("Application attempt " + application.getApplicationAttemptId() + + " released container " + container.getId() + " on node: " + node + + " with event: " + event); + } } private synchronized void addNode(RMNode node) {