diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 33e8a1f..782bc32 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -375,6 +375,12 @@ private static void addDeprecatedKeys() { public static final int DEFAULT_RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE = 10; + /** Whether permit RM release container's resource before NM. */ + public static final String RM_RELEASE_CONTAINER_RESOURCE_BEFORE_NM = + RM_PREFIX + "release_container_resource_before_nm"; + public static final boolean DEFAULT_RM_RELEASE_CONTAINER_RESOURCE_BEFORE_NM = + true; + /** * The setting that controls whether yarn system metrics is published on the * timeline server or not by RM. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index b64c834..ab7697d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -129,4 +129,6 @@ void setRMApplicationHistoryWriter( PlacementManager getQueuePlacementManager(); void setQueuePlacementManager(PlacementManager placementMgr); + + boolean canReleaseContainerResourceBeforeNM(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index 840cea7..e81be94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -72,6 +72,8 @@ private RMApplicationHistoryWriter rmApplicationHistoryWriter; private SystemMetricsPublisher systemMetricsPublisher; + private boolean releaseContainerResourceBeforeNM; + /** * Default constructor. To be used in conjunction with setter methods for * individual fields. @@ -448,4 +450,15 @@ public PlacementManager getQueuePlacementManager() { public void setQueuePlacementManager(PlacementManager placementMgr) { this.activeServiceContext.setQueuePlacementManager(placementMgr); } + + @Override + public boolean canReleaseContainerResourceBeforeNM() { + return releaseContainerResourceBeforeNM; + } + + void setReleaseContainerResourceBeforeNM( + boolean canReleaseContainerResourceBeforeNM) { + this.releaseContainerResourceBeforeNM = + canReleaseContainerResourceBeforeNM; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index d6d9629..36fbad1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -231,7 +231,14 @@ protected void serviceInit(Configuration conf) throws Exception { if (this.rmContext.isHAEnabled()) { HAUtil.verifyAndSetConfiguration(this.conf); } - + + // Set whether RM could releases container's resource before NM + this.rmContext.setReleaseContainerResourceBeforeNM( + conf.getBoolean( + YarnConfiguration.RM_RELEASE_CONTAINER_RESOURCE_BEFORE_NM, + YarnConfiguration.DEFAULT_RM_RELEASE_CONTAINER_RESOURCE_BEFORE_NM) + ); + // Set UGI and do login // If security is enabled, use login user // If security is not enabled, use current user diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java index 7a1ba74..7a3950f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRunningOnNodeEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerResourceReleasedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent; @@ -930,6 +931,14 @@ private void handleContainerStatus(List containerStatuses) { for (ContainerStatus remoteContainer : containerStatuses) { ContainerId containerId = remoteContainer.getContainerId(); + if (remoteContainer.getState() == ContainerState.COMPLETE + && !context.canReleaseContainerResourceBeforeNM()) { + // Released container's resource + context.getDispatcher().getEventHandler().handle( + new ContainerResourceReleasedSchedulerEvent( + remoteContainer.getContainerId(), nodeId)); + } + // Don't bother with containers already scheduled for cleanup, or for // applications already killed. The scheduler doens't need to know any // more about this container diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java index f03663a..5fd522d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java @@ -221,6 +221,37 @@ public synchronized void releaseContainer(Container container) { + " available" + ", release resources=" + true); } + /** + * Release an allocated container on this node. + * + * @param containerId + * containerId to be released + */ + public synchronized void releaseContainer(ContainerId containerId) { + if (!isValidContainer(containerId)) { + LOG.error("Invalid container released " + containerId); + return; + } + + RMContainer rmContainer = launchedContainers.remove(containerId); + if (rmContainer == null) { + if (LOG.isDebugEnabled()) { + LOG.debug(containerId + "'s resource has already been released"); + } + return; + } + + Container container = rmContainer.getContainer(); + /* remove the containers from the nodemanger */ + updateResource(container); + + LOG.info("Released container " + container.getId() + " of capacity " + + container.getResource() + " on host " + rmNode.getNodeAddress() + + ", which currently has " + numContainers + " containers, " + + getUsedResource() + " used and " + getAvailableResource() + + " available" + ", release resources=" + true); + } + private synchronized void addAvailableResource(Resource resource) { if (resource == null) { LOG.error("Invalid resource addition of null resource for " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index ad5c76c..1574297 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -111,6 +111,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerRescheduledEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerResourceReleasedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; @@ -1326,6 +1327,17 @@ public void handle(SchedulerEvent event) { recoverResourceRequestForContainer(container); } break; + case CONTAINER_RESOURCE_RELEASED: + { + ContainerResourceReleasedSchedulerEvent + containerResourceReleasedSchedulerEvent = + (ContainerResourceReleasedSchedulerEvent) event; + FiCaSchedulerNode node = getNode( + containerResourceReleasedSchedulerEvent.getNodeId()); + node.releaseContainer( + containerResourceReleasedSchedulerEvent.getContainerId()); + } + break; default: LOG.error("Invalid eventtype " + event.getType() + ". Ignoring!"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index b43f658..39d3df4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -1112,8 +1112,10 @@ public void completedContainer(Resource clusterResource, removed = application.containerCompleted(rmContainer, containerStatus, event, node.getPartition()); - - node.releaseContainer(container); + + if (scheduler.getRMContext().canReleaseContainerResourceBeforeNM()) { + node.releaseContainer(container); + } } // Book-keeping diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/ContainerResourceReleasedSchedulerEvent.java similarity index 58% copy from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java copy to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/ContainerResourceReleasedSchedulerEvent.java index 40dd66b..71d7ffd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/ContainerResourceReleasedSchedulerEvent.java @@ -18,31 +18,25 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.event; -public enum SchedulerEventType { - - // Source: Node - NODE_ADDED, - NODE_REMOVED, - NODE_UPDATE, - NODE_RESOURCE_UPDATE, - NODE_LABELS_UPDATE, - - // Source: RMApp - APP_ADDED, - APP_REMOVED, - - // Source: RMAppAttempt - APP_ATTEMPT_ADDED, - APP_ATTEMPT_REMOVED, - - // Source: ContainerAllocationExpirer - CONTAINER_EXPIRED, - - // Source: RMContainer - CONTAINER_RESCHEDULED, - - // Source: SchedulingEditPolicy - DROP_RESERVATION, - PREEMPT_CONTAINER, - KILL_CONTAINER +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; + +public class ContainerResourceReleasedSchedulerEvent extends SchedulerEvent { + private ContainerId containerId; + private NodeId nodeId; + + public ContainerResourceReleasedSchedulerEvent(ContainerId containerId, + NodeId nodeId) { + super(SchedulerEventType.CONTAINER_RESOURCE_RELEASED); + this.containerId = containerId; + this.nodeId = nodeId; + } + + public ContainerId getContainerId() { + return containerId; + } + + public NodeId getNodeId() { + return nodeId; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java index 40dd66b..dd16570 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java @@ -26,6 +26,7 @@ NODE_UPDATE, NODE_RESOURCE_UPDATE, NODE_LABELS_UPDATE, + CONTAINER_RESOURCE_RELEASED, // Source: RMApp APP_ADDED, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 3a39799..16bcab6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -77,6 +77,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerRescheduledEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerResourceReleasedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent; @@ -835,7 +836,9 @@ protected synchronized void completedContainer(RMContainer rmContainer, application.unreserve(rmContainer.getReservedPriority(), node); } else { application.containerCompleted(rmContainer, containerStatus, event); - node.releaseContainer(container); + if (rmContext.canReleaseContainerResourceBeforeNM()) { + node.releaseContainer(container); + } updateRootQueueMetrics(); } @@ -1259,6 +1262,18 @@ public void handle(SchedulerEvent event) { RMContainer container = containerRescheduledEvent.getContainer(); recoverResourceRequestForContainer(container); break; + case CONTAINER_RESOURCE_RELEASED: + if (!(event instanceof ContainerResourceReleasedSchedulerEvent)) { + throw new RuntimeException("Unexpected event type: " + event); + } + ContainerResourceReleasedSchedulerEvent + containerResourceReleasedSchedulerEvent = + (ContainerResourceReleasedSchedulerEvent)event; + FSSchedulerNode node = getFSSchedulerNode( + containerResourceReleasedSchedulerEvent.getNodeId()); + node.releaseContainer( + containerResourceReleasedSchedulerEvent.getContainerId()); + break; default: LOG.error("Unknown event arrived at FairScheduler: " + event.toString()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 99760df..1f48212 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -86,6 +86,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerRescheduledEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerResourceReleasedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent; @@ -868,6 +869,17 @@ public void handle(SchedulerEvent event) { recoverResourceRequestForContainer(container); } break; + case CONTAINER_RESOURCE_RELEASED: + { + ContainerResourceReleasedSchedulerEvent + containerResourceReleasedSchedulerEvent = + (ContainerResourceReleasedSchedulerEvent) event; + FiCaSchedulerNode node = getNode( + containerResourceReleasedSchedulerEvent.getNodeId()); + node.releaseContainer( + containerResourceReleasedSchedulerEvent.getContainerId()); + } + break; default: LOG.error("Invalid eventtype " + event.getType() + ". Ignoring!"); } @@ -905,8 +917,10 @@ protected synchronized void completedContainer(RMContainer rmContainer, RMNodeLabelsManager.NO_LABEL); // Inform the node - node.releaseContainer(container); - + if (rmContext.canReleaseContainerResourceBeforeNM()) { + node.releaseContainer(container); + } + // Update total usage Resources.subtractFrom(usedResource, container.getResource());