From a8153ba53d7470b2a451fe688e20b9121b831a93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=B7=A8=E4=B8=B0?= <920347627@qq.com> Date: Fri, 6 Nov 2020 16:33:59 +0800 Subject: [PATCH 8/8] =?UTF-8?q?=E4=BF=AE=E6=94=B9RM=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E7=9A=84=E6=89=80=E6=9C=89ReadWriteLock=E4=B8=BAsynchronized?= =?UTF-8?q?=E9=94=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../SchedulerApplicationAttempt.java | 219 +++++------------- .../common/fica/FiCaSchedulerApp.java | 125 +++------- .../scheduler/fair/FSAppAttempt.java | 71 ++---- .../scheduler/fair/FSPreemptionThread.java | 7 +- .../scheduler/fair/FairScheduler.java | 162 +++---------- .../scheduler/fifo/FifoAppAttempt.java | 7 +- 6 files changed, 153 insertions(+), 438 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 767df6d..8d930ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -31,7 +31,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.time.DateUtils; @@ -195,8 +194,6 @@ private RMAppAttempt appAttempt; - protected ReentrantReadWriteLock.ReadLock readLock; - protected ReentrantReadWriteLock.WriteLock writeLock; private Map applicationSchedulingEnvs = new HashMap<>(); @@ -233,9 +230,6 @@ public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, this.appSchedulingInfo = new AppSchedulingInfo(applicationAttemptId, user, queue, abstractUsersManager, rmContext.getEpoch(), attemptResourceUsage, applicationSchedulingEnvs, rmContext); - ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - readLock = lock.readLock(); - writeLock = lock.writeLock(); } public void setOpportunisticContainerContext( @@ -252,13 +246,9 @@ public void setOpportunisticContainerContext( * Get the live containers of the application. * @return live containers of the application */ - public Collection getLiveContainers() { - try { - readLock.lock(); + public synchronized Collection getLiveContainers() { + return new ArrayList<>(liveContainers.values()); - } finally { - readLock.unlock(); - } } public AppSchedulingInfo getAppSchedulingInfo() { @@ -305,30 +295,21 @@ public long getNewContainerId() { return appSchedulingInfo.getSchedulerKeys(); } - public PendingAsk getPendingAsk( + public synchronized PendingAsk getPendingAsk( SchedulerRequestKey schedulerKey, String resourceName) { - try { - readLock.lock(); return appSchedulingInfo.getPendingAsk(schedulerKey, resourceName); - } finally { - readLock.unlock(); - } + } public int getOutstandingAsksCount(SchedulerRequestKey schedulerKey) { return getOutstandingAsksCount(schedulerKey, ResourceRequest.ANY); } - public int getOutstandingAsksCount(SchedulerRequestKey schedulerKey, + public synchronized int getOutstandingAsksCount(SchedulerRequestKey schedulerKey, String resourceName) { - try { - readLock.lock(); AppPlacementAllocator ap = appSchedulingInfo.getAppPlacementAllocator( schedulerKey); return ap == null ? 0 : ap.getOutstandingAsksCount(resourceName); - } finally { - readLock.unlock(); - } } public String getQueueName() { @@ -367,10 +348,9 @@ public RMContainer getRMContainer(ContainerId id) { return liveContainers.get(id); } - public void addRMContainer( + public synchronized void addRMContainer( ContainerId id, RMContainer rmContainer) { - try { - writeLock.lock(); + if (!getApplicationAttemptId().equals( rmContainer.getApplicationAttemptId()) && !liveContainers.containsKey(id)) { @@ -387,14 +367,10 @@ public void addRMContainer( this.attemptResourceUsageAllocatedRemotely.incUsed( rmContainer.getAllocatedResource()); } - } finally { - writeLock.unlock(); - } } - public void removeRMContainer(ContainerId containerId) { - try { - writeLock.lock(); + public synchronized void removeRMContainer(ContainerId containerId) { + RMContainer rmContainer = liveContainers.remove(containerId); if (rmContainer != null) { if (rmContainer.getExecutionType() == ExecutionType.OPPORTUNISTIC) { @@ -406,9 +382,6 @@ public void removeRMContainer(ContainerId containerId) { .decUsed(rmContainer.getAllocatedResource()); } } - } finally { - writeLock.unlock(); - } } protected void resetReReservations( @@ -444,17 +417,14 @@ public Queue getQueue() { return queue; } - public boolean updateResourceRequests( + public synchronized boolean updateResourceRequests( List requests) { - try { - writeLock.lock(); + if (!isStopped) { return appSchedulingInfo.updateResourceRequests(requests, false); } return false; - } finally { - writeLock.unlock(); - } + } public boolean updateSchedulingRequests( @@ -463,39 +433,30 @@ public boolean updateSchedulingRequests( return false; } - try { - writeLock.lock(); + synchronized (this){ if (!isStopped) { return appSchedulingInfo.updateSchedulingRequests(requests, false); } return false; - } finally { - writeLock.unlock(); } } - public void recoverResourceRequestsForContainer( + public synchronized void recoverResourceRequestsForContainer( ContainerRequest containerRequest) { - try { - writeLock.lock(); + if (!isStopped) { appSchedulingInfo.updateResourceRequests( containerRequest.getResourceRequests(), true); } - } finally { - writeLock.unlock(); - } + } - public void stop(RMAppAttemptState rmAppAttemptFinalState) { - try { - writeLock.lock(); + public synchronized void stop(RMAppAttemptState rmAppAttemptFinalState) { + // Cleanup all scheduling information isStopped = true; appSchedulingInfo.stop(); - } finally { - writeLock.unlock(); - } + } public boolean isStopped() { @@ -506,26 +467,21 @@ public boolean isStopped() { * Get the list of reserved containers * @return All of the reserved containers. */ - public List getReservedContainers() { + public synchronized List getReservedContainers() { List list = new ArrayList<>(); - try { - readLock.lock(); + for (Entry> e : this.reservedContainers.entrySet()) { list.addAll(e.getValue().values()); } return list; - } finally { - readLock.unlock(); - } } - public boolean reserveIncreasedContainer(SchedulerNode node, + public synchronized boolean reserveIncreasedContainer(SchedulerNode node, SchedulerRequestKey schedulerKey, RMContainer rmContainer, Resource reservedResource) { - try { - writeLock.lock(); + if (commonReserve(node, schedulerKey, rmContainer, reservedResource)) { attemptResourceUsage.incReserved(node.getPartition(), reservedResource); // succeeded @@ -533,9 +489,7 @@ public boolean reserveIncreasedContainer(SchedulerNode node, } return false; - } finally { - writeLock.unlock(); - } + } @@ -570,11 +524,10 @@ private boolean commonReserve(SchedulerNode node, return true; } - public RMContainer reserve(SchedulerNode node, + public synchronized RMContainer reserve(SchedulerNode node, SchedulerRequestKey schedulerKey, RMContainer rmContainer, Container container) { - try { - writeLock.lock(); + // Create RMContainer if necessary if (rmContainer == null) { rmContainer = new RMContainerImpl(container, schedulerKey, @@ -596,9 +549,6 @@ public RMContainer reserve(SchedulerNode node, commonReserve(node, schedulerKey, rmContainer, container.getResource()); return rmContainer; - } finally { - writeLock.unlock(); - } } @@ -615,23 +565,19 @@ public Resource getHeadroom() { return resourceLimit; } - public int getNumReservedContainers( + public synchronized int getNumReservedContainers( SchedulerRequestKey schedulerKey) { - try { - readLock.lock(); + Map map = this.reservedContainers.get( schedulerKey); return (map == null) ? 0 : map.size(); - } finally { - readLock.unlock(); - } + } @SuppressWarnings("unchecked") - public void containerLaunchedOnNode(ContainerId containerId, + public synchronized void containerLaunchedOnNode(ContainerId containerId, NodeId nodeId) { - try { - writeLock.lock(); + // Inform the container RMContainer rmContainer = getRMContainer(containerId); if (rmContainer == null) { @@ -643,15 +589,11 @@ public void containerLaunchedOnNode(ContainerId containerId, rmContainer.handle( new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED)); - } finally { - writeLock.unlock(); - } } public void showRequests() { if (LOG.isDebugEnabled()) { - try { - readLock.lock(); + synchronized (this) { for (SchedulerRequestKey schedulerKey : getSchedulerKeys()) { AppPlacementAllocator ap = getAppPlacementAllocator(schedulerKey); if (ap != null && @@ -662,8 +604,6 @@ public void showRequests() { ap.showRequests(); } } - } finally { - readLock.unlock(); } } } @@ -761,14 +701,11 @@ private void updateNMToken(Container container) { * RegisterApplicationMasterResponse#containersFromPreviousAttempts * . */ - List pullContainersToTransfer() { - try { - writeLock.lock(); + synchronized List pullContainersToTransfer() { + recoveredPreviousAttemptContainers.clear(); return new ArrayList<>(liveContainers.values()); - } finally { - writeLock.unlock(); - } + } /** @@ -776,9 +713,8 @@ private void updateNMToken(Container container) { * the AM had registered. They are reported to the AM in the * AllocateResponse#containersFromPreviousAttempts. */ - public List pullPreviousAttemptContainers() { - try { - writeLock.lock(); + public synchronized List pullPreviousAttemptContainers() { + if (recoveredPreviousAttemptContainers.isEmpty()) { return null; } @@ -787,17 +723,14 @@ private void updateNMToken(Container container) { recoveredPreviousAttemptContainers.clear(); updateNMTokens(returnContainerList); return returnContainerList; - } finally { - writeLock.unlock(); - } + } // Create container token and update NMToken altogether, if either of them fails for // some reason like DNS unavailable, do not return this container and keep it // in the newlyAllocatedContainers waiting to be refetched. - public List pullNewlyAllocatedContainers() { - try { - writeLock.lock(); + public synchronized List pullNewlyAllocatedContainers() { + List returnContainerList = new ArrayList( newlyAllocatedContainers.size()); @@ -815,9 +748,6 @@ private void updateNMToken(Container container) { } } return returnContainerList; - } finally { - writeLock.unlock(); - } } public synchronized void addToNewlyDemotedContainers(ContainerId containerId, @@ -912,8 +842,7 @@ protected synchronized void addToNewlyAllocatedContainers( || ContainerUpdateType.PROMOTE_EXECUTION_TYPE == updateTpe)) { return updatedContainers; } - try { - writeLock.lock(); + synchronized (this) { Iterator> i = newlyUpdatedContainers.entrySet().iterator(); while (i.hasNext()) { @@ -954,20 +883,14 @@ protected synchronized void addToNewlyAllocatedContainers( tempIter.remove(); } return updatedContainers; - } finally { - writeLock.unlock(); } } - public List pullUpdatedNMTokens() { - try { - writeLock.lock(); + public synchronized List pullUpdatedNMTokens() { + List returnList = new ArrayList<>(updatedNMTokens); updatedNMTokens.clear(); return returnList; - } finally { - writeLock.unlock(); - } } @@ -977,10 +900,9 @@ public boolean isWaitingForAMContainer() { return (!unmanagedAM && appAttempt.getMasterContainer() == null); } - public void updateBlacklist(List blacklistAdditions, + public synchronized void updateBlacklist(List blacklistAdditions, List blacklistRemovals) { - try { - writeLock.lock(); + if (!isStopped) { if (isWaitingForAMContainer()) { // The request is for the AM-container, and the AM-container is @@ -993,20 +915,14 @@ public void updateBlacklist(List blacklistAdditions, blacklistAdditions, blacklistRemovals); } } - } finally { - writeLock.unlock(); - } } - public boolean isPlaceBlacklisted(String resourceName) { - try { - readLock.lock(); + public synchronized boolean isPlaceBlacklisted(String resourceName) { + boolean forAMContainer = isWaitingForAMContainer(); return this.appSchedulingInfo.isPlaceBlacklisted(resourceName, forAMContainer); - } finally { - readLock.unlock(); - } + } public int addMissedNonPartitionedRequestSchedulingOpportunity( @@ -1102,9 +1018,8 @@ private AggregateAppResourceUsage getRunningAggregateAppResourceUsage() { return new AggregateAppResourceUsage(lastResourceSecondsMap); } - public ApplicationResourceUsageReport getResourceUsageReport() { - try { - writeLock.lock(); + public synchronized ApplicationResourceUsageReport getResourceUsageReport() { + AggregateAppResourceUsage runningResourceUsage = getRunningAggregateAppResourceUsage(); Resource usedResourceClone = Resources.clone( @@ -1137,9 +1052,7 @@ public ApplicationResourceUsageReport getResourceUsageReport() { Resources.add(usedResourceClone, reservedResourceClone), runningResourceUsage.getResourceUsageSecondsMap(), queueUsagePerc, clusterUsagePerc, preemptedResourceSecondsMaps); - } finally { - writeLock.unlock(); - } + } @VisibleForTesting @@ -1152,10 +1065,9 @@ public ApplicationResourceUsageReport getResourceUsageReport() { return this.lastScheduledContainer; } - public void transferStateFromPreviousAttempt( + public synchronized void transferStateFromPreviousAttempt( SchedulerApplicationAttempt appAttempt) { - try { - writeLock.lock(); + this.liveContainers = appAttempt.getLiveContainersMap(); // this.reReservations = appAttempt.reReservations; this.attemptResourceUsage.copyAllUsed(appAttempt.attemptResourceUsage); @@ -1166,14 +1078,11 @@ public void transferStateFromPreviousAttempt( this.lastScheduledContainer = appAttempt.getLastScheduledContainer(); this.appSchedulingInfo.transferStateFromPreviousAppSchedulingInfo( appAttempt.appSchedulingInfo); - } finally { - writeLock.unlock(); - } + } - public void move(Queue newQueue) { - try { - writeLock.lock(); + public synchronized void move(Queue newQueue) { + QueueMetrics oldMetrics = queue.getMetrics(); QueueMetrics newMetrics = newQueue.getMetrics(); String newQueueName = newQueue.getQueueName(); @@ -1202,15 +1111,12 @@ public void move(Queue newQueue) { appSchedulingInfo.move(newQueue); } this.queue = newQueue; - } finally { - writeLock.unlock(); - } + } - public void recoverContainer(SchedulerNode node, + public synchronized void recoverContainer(SchedulerNode node, RMContainer rmContainer) { - try { - writeLock.lock(); + // recover app scheduling info appSchedulingInfo.recoverContainer(rmContainer, node.getPartition()); @@ -1230,9 +1136,7 @@ public void recoverContainer(SchedulerNode node, // newlyAllocatedContainers.add(rmContainer); // schedulingOpportunities // lastScheduledContainer - } finally { - writeLock.unlock(); - } + } public void incNumAllocatedContainers(NodeType containerType, @@ -1402,9 +1306,6 @@ protected void getActivedAppDiagnosticMessage( // queue's resource usage for specific partition } - public ReentrantReadWriteLock.WriteLock getWriteLock() { - return writeLock; - } @Override public boolean isRecovering() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 6a5af81..fdbfb05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -25,7 +25,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -174,11 +173,10 @@ public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId, activitiesManager); } - public boolean containerCompleted(RMContainer rmContainer, + public synchronized boolean containerCompleted(RMContainer rmContainer, ContainerStatus containerStatus, RMContainerEventType event, String partition) { - try { - writeLock.lock(); + ContainerId containerId = rmContainer.getContainerId(); // Remove from the list of containers @@ -208,15 +206,11 @@ public boolean containerCompleted(RMContainer rmContainer, lastMemoryAggregateAllocationUpdateTime = -1; return true; - } finally { - writeLock.unlock(); - } } - public RMContainer allocate(FiCaSchedulerNode node, + public synchronized RMContainer allocate(FiCaSchedulerNode node, SchedulerRequestKey schedulerKey, Container container) { - try { - readLock.lock(); + if (isStopped) { return null; @@ -248,9 +242,7 @@ public RMContainer allocate(FiCaSchedulerNode node, updateAMContainerDiagnostics(AMState.ASSIGNED, null); return rmContainer; - } finally { - readLock.unlock(); - } + } private boolean rmContainerInFinalState(RMContainer rmContainer) { @@ -407,8 +399,7 @@ public boolean accept(Resource cluster, ContainerRequest containerRequest = null; boolean reReservation = false; - try { - readLock.lock(); + synchronized (this) { // First make sure no container in release list in final state if (anyContainerInFinalState(request)) { @@ -505,8 +496,6 @@ public boolean accept(Resource cluster, } } } - } finally { - readLock.unlock(); } // Skip check parent if this is a re-reservation container @@ -530,8 +519,7 @@ public boolean apply(Resource cluster, ResourceCommitRequest request, boolean updatePending) { boolean reReservation = false; - try { - writeLock.lock(); + synchronized (this) { // If we allocated something if (request.anythingAllocatedOrReserved()) { @@ -641,10 +629,7 @@ public boolean apply(Resource cluster, ResourceCommitRequest getTotalPendingRequestsPerPartition() { - try { - readLock.lock(); + public synchronized Map getTotalPendingRequestsPerPartition() { + Map ret = new HashMap<>(); for (SchedulerRequestKey schedulerKey : appSchedulingInfo @@ -735,22 +716,17 @@ private boolean internalUnreserve(FiCaSchedulerNode node, } return ret; - } finally { - readLock.unlock(); - } + } - public void markContainerForPreemption(ContainerId cont) { - try { - writeLock.lock(); + public synchronized void markContainerForPreemption(ContainerId cont) { + // ignore already completed containers if (liveContainers.containsKey(cont)) { containersToPreempt.add(cont); } - } finally { - writeLock.unlock(); - } + } /** @@ -763,10 +739,9 @@ public void markContainerForPreemption(ContainerId cont) { * @param minimumAllocation minimumAllocation * @return an allocation */ - public Allocation getAllocation(ResourceCalculator resourceCalculator, + public synchronized Allocation getAllocation(ResourceCalculator resourceCalculator, Resource clusterResource, Resource minimumAllocation) { - try { - writeLock.lock(); + Set currentContPreemption = Collections.unmodifiableSet( new HashSet(containersToPreempt)); containersToPreempt.clear(); @@ -795,9 +770,7 @@ public Allocation getAllocation(ResourceCalculator resourceCalculator, newlyIncreasedContainers, newlyDecreasedContainers, newlyPromotedContainers, newlyDemotedContainers, previousAttemptContainers); - } finally { - writeLock.unlock(); - } + } @VisibleForTesting @@ -832,40 +805,31 @@ public NodeId getNodeIdToUnreserve(SchedulerRequestKey schedulerKey, return null; } - public void setHeadroomProvider( + public synchronized void setHeadroomProvider( CapacityHeadroomProvider headroomProvider) { - try { - writeLock.lock(); + this.headroomProvider = headroomProvider; - } finally { - writeLock.unlock(); - } + } @Override - public Resource getHeadroom() { - try { - readLock.lock(); + public synchronized Resource getHeadroom() { + if (headroomProvider != null) { return headroomProvider.getHeadroom(); } return super.getHeadroom(); - } finally { - readLock.unlock(); - } + } @Override - public void transferStateFromPreviousAttempt( + public synchronized void transferStateFromPreviousAttempt( SchedulerApplicationAttempt appAttempt) { - try { - writeLock.lock(); + super.transferStateFromPreviousAttempt(appAttempt); this.headroomProvider = ((FiCaSchedulerApp) appAttempt).headroomProvider; - } finally { - writeLock.unlock(); - } + } public void reserve(SchedulerRequestKey schedulerKey, FiCaSchedulerNode node, @@ -885,10 +849,9 @@ public void reserve(SchedulerRequestKey schedulerKey, FiCaSchedulerNode node, } @VisibleForTesting - public RMContainer findNodeToUnreserve(FiCaSchedulerNode node, + public synchronized RMContainer findNodeToUnreserve(FiCaSchedulerNode node, SchedulerRequestKey schedulerKey, Resource minimumUnreservedResource) { - try { - readLock.lock(); + // need to unreserve some other container first NodeId idToUnreserve = getNodeIdToUnreserve(schedulerKey, minimumUnreservedResource, rc); @@ -917,9 +880,7 @@ public RMContainer findNodeToUnreserve(FiCaSchedulerNode node, nodeToUnreserve.getReservedContainer().getReservedResource()); return nodeToUnreserve.getReservedContainer(); - } finally { - readLock.unlock(); - } + } public LeafQueue getCSLeafQueue() { @@ -1069,12 +1030,8 @@ public void updateNodeInfoForAMDiagnostics(FiCaSchedulerNode node) { * Capacity Scheduler. */ @Override - public ApplicationResourceUsageReport getResourceUsageReport() { - try { - // Use write lock here because - // SchedulerApplicationAttempt#getResourceUsageReport updated fields - // TODO: improve this - writeLock.lock(); + public synchronized ApplicationResourceUsageReport getResourceUsageReport() { + ApplicationResourceUsageReport report = super.getResourceUsageReport(); Resource cluster = rmContext.getScheduler().getClusterResource(); Resource totalPartitionRes = @@ -1093,15 +1050,11 @@ public ApplicationResourceUsageReport getResourceUsageReport() { report.setQueueUsagePercentage(queueUsagePerc); } return report; - } finally { - writeLock.unlock(); - } - } - public ReentrantReadWriteLock.WriteLock getWriteLock() { - return this.writeLock; } + + public void addToBeRemovedIncreaseRequest( SchedContainerChangeRequest request) { toBeRemovedIncRequests.put(request.getContainerId(), request); @@ -1135,10 +1088,9 @@ public boolean equals(Object o) { * * @return succeeded or not */ - public boolean moveReservation(RMContainer reservedContainer, + public synchronized boolean moveReservation(RMContainer reservedContainer, FiCaSchedulerNode sourceNode, FiCaSchedulerNode targetNode) { - try { - writeLock.lock(); + if (!sourceNode.getPartition().equals(targetNode.getPartition())) { if (LOG.isDebugEnabled()) { LOG.debug( @@ -1199,8 +1151,5 @@ public boolean moveReservation(RMContainer reservedContainer, return true; } - } finally { - writeLock.unlock(); - } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 0305702..e1c19f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -129,10 +129,8 @@ public QueueMetrics getMetrics() { return queue.getMetrics(); } - void containerCompleted(RMContainer rmContainer, + synchronized void containerCompleted(RMContainer rmContainer, ContainerStatus containerStatus, RMContainerEventType event) { - try { - writeLock.lock(); Container container = rmContainer.getContainer(); ContainerId containerId = container.getId(); @@ -173,15 +171,11 @@ void containerCompleted(RMContainer rmContainer, // Clear resource utilization metrics cache. lastMemoryAggregateAllocationUpdateTime = -1; - } finally { - writeLock.unlock(); - } } - private void unreserveInternal( + private synchronized void unreserveInternal( SchedulerRequestKey schedulerKey, FSSchedulerNode node) { - try { - writeLock.lock(); + Map reservedContainers = this.reservedContainers.get( schedulerKey); RMContainer reservedContainer = reservedContainers.remove( @@ -202,9 +196,7 @@ private void unreserveInternal( + " at priority " + schedulerKey.getPriority() + "; currentReservation " + this.attemptResourceUsage .getReserved()); - } finally { - writeLock.unlock(); - } + } private void subtractResourcesOnBlacklistedNodes( @@ -283,8 +275,7 @@ NodeType getAllowedLocalityLevel( return NodeType.OFF_SWITCH; } - try { - writeLock.lock(); + synchronized (this) { // Default level is NODE_LOCAL if (!allowedLocalityLevel.containsKey(schedulerKey)) { @@ -330,8 +321,6 @@ NodeType getAllowedLocalityLevel( } } return allowedLocalityLevel.get(schedulerKey); - } finally { - writeLock.unlock(); } } @@ -353,8 +342,7 @@ NodeType getAllowedLocalityLevelByTime( return NodeType.OFF_SWITCH; } - try { - writeLock.lock(); + synchronized (this) { // default level is NODE_LOCAL if (!allowedLocalityLevel.containsKey(schedulerKey)) { @@ -413,19 +401,15 @@ NodeType getAllowedLocalityLevelByTime( } } return allowedLocalityLevel.get(schedulerKey); - } finally { - writeLock.unlock(); } } - public RMContainer allocate(NodeType type, FSSchedulerNode node, + public synchronized RMContainer allocate(NodeType type, FSSchedulerNode node, SchedulerRequestKey schedulerKey, PendingAsk pendingAsk, Container reservedContainer) { RMContainer rmContainer; Container container; - try { - writeLock.lock(); // Update allowed locality level NodeType allowed = allowedLocalityLevel.get(schedulerKey); if (allowed != null) { @@ -481,9 +465,7 @@ public RMContainer allocate(NodeType type, FSSchedulerNode node, RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER, "SchedulerApp", getApplicationId(), container.getId(), container.getResource()); - } finally { - writeLock.unlock(); - } + return rmContainer; } @@ -495,15 +477,12 @@ public RMContainer allocate(NodeType type, FSSchedulerNode node, * @param schedulerKey Scheduler Key * @param level NodeType */ - void resetAllowedLocalityLevel( + synchronized void resetAllowedLocalityLevel( SchedulerRequestKey schedulerKey, NodeType level) { NodeType old; - try { - writeLock.lock(); + old = allowedLocalityLevel.put(schedulerKey, level); - } finally { - writeLock.unlock(); - } + LOG.info("Raising locality level from " + old + " to " + level + " at " + " priority " + schedulerKey.getPriority()); @@ -664,9 +643,6 @@ private Container createContainer(FSSchedulerNode node, Resource capability, @Override public synchronized void recoverContainer(SchedulerNode node, RMContainer rmContainer) { - try { - writeLock.lock(); - super.recoverContainer(node, rmContainer); if (!rmContainer.getState().equals(RMContainerState.COMPLETED)) { @@ -682,9 +658,6 @@ public synchronized void recoverContainer(SchedulerNode node, getQueue().addAMResourceUsage(resource); setAmRunning(true); } - } finally { - writeLock.unlock(); - } } /** @@ -772,36 +745,29 @@ public void unreserve(SchedulerRequestKey schedulerKey, getUser(), rmContainer.getContainer().getResource()); } - private void setReservation(SchedulerNode node) { + private synchronized void setReservation(SchedulerNode node) { String rackName = node.getRackName() == null ? "NULL" : node.getRackName(); - try { - writeLock.lock(); + Set rackReservations = reservations.get(rackName); if (rackReservations == null) { rackReservations = new HashSet<>(); reservations.put(rackName, rackReservations); } rackReservations.add(node.getNodeName()); - } finally { - writeLock.unlock(); - } + } - private void clearReservation(SchedulerNode node) { + private synchronized void clearReservation(SchedulerNode node) { String rackName = node.getRackName() == null ? "NULL" : node.getRackName(); - try { - writeLock.lock(); Set rackReservations = reservations.get(rackName); if (rackReservations != null) { rackReservations.remove(node.getNodeName()); } - } finally { - writeLock.unlock(); - } + } int getNumReservations(String rackName, boolean isAny) { @@ -957,8 +923,7 @@ private Resource assignContainer(FSSchedulerNode node, boolean reserved) { // For each priority, see if we can schedule a node local, rack local // or off-switch request. Rack of off-switch requests may be delayed // (not scheduled) in order to promote better locality. - try { - writeLock.lock(); + synchronized (this) { // TODO (wandga): All logics in this method should be added to // SchedulerPlacement#canDelayTo which is independent from scheduler. @@ -1055,8 +1020,6 @@ private Resource assignContainer(FSSchedulerNode node, boolean reserved) { + this.attemptId); } } - } finally { - writeLock.unlock(); } return Resources.none(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java index c32565f..02969a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java @@ -46,7 +46,6 @@ private final long warnTimeBeforeKill; private final long delayBeforeNextStarvationCheck; private final Timer preemptionTimer; - private final Lock schedulerReadLock; @SuppressWarnings("deprecation") FSPreemptionThread(FairScheduler scheduler) { @@ -66,7 +65,6 @@ : 4 * scheduler.getNMHeartbeatInterval()); // 4 heartbeats delayBeforeNextStarvationCheck = warnTimeBeforeKill + allocDelay + fsConf.getWaitTimeBeforeNextStarvationCheck(); - schedulerReadLock = scheduler.getSchedulerReadLock(); } @Override @@ -76,11 +74,8 @@ public void run() { FSAppAttempt starvedApp = context.getStarvedApps().take(); // Hold the scheduler readlock so this is not concurrent with the // update thread. - schedulerReadLock.lock(); - try { + synchronized (this){ preemptContainers(identifyContainersToPreempt(starvedApp)); - } finally { - schedulerReadLock.unlock(); } starvedApp.preemptionTriggered(delayBeforeNextStarvationCheck); } catch (InterruptedException e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index caa6aa0..dc817c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -104,8 +104,7 @@ import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; + /** * A scheduler that schedules resources between a set of queues. The scheduler @@ -338,20 +337,16 @@ public void update() { FSQueue rootQueue = queueMgr.getRootQueue(); // Update demands and fairshares - writeLock.lock(); - try { + synchronized (this){ // Recursively update demands for all queues rootQueue.updateDemand(); rootQueue.update(getClusterResource()); // Update metrics updateRootQueueMetrics(); - } finally { - writeLock.unlock(); } - readLock.lock(); - try { + synchronized (this){ // Update starvation stats and identify starved applications if (shouldAttemptPreemption()) { for (FSLeafQueue queue : queueMgr.getLeafQueues()) { @@ -366,8 +361,6 @@ public void update() { dumpSchedulerState(); } } - } finally { - readLock.unlock(); } fsOpDurations.addUpdateThreadRunDuration(getClock().getTime() - start); } @@ -469,8 +462,7 @@ protected void addApplication(ApplicationId applicationId, return; } - try { - writeLock.lock(); + synchronized (this){ RMApp rmApp = rmContext.getRMApps().get(applicationId); FSLeafQueue queue = assignToQueue(rmApp, queueName, user); if (queue == null) { @@ -516,20 +508,17 @@ protected void addApplication(ApplicationId applicationId, rmContext.getDispatcher().getEventHandler().handle( new RMAppEvent(applicationId, RMAppEventType.APP_ACCEPTED)); } - } finally { - writeLock.unlock(); } } /** * Add a new application attempt to the scheduler. */ - protected void addApplicationAttempt( + protected synchronized void addApplicationAttempt( ApplicationAttemptId applicationAttemptId, boolean transferStateFromPreviousAttempt, boolean isAttemptRecovering) { - try { - writeLock.lock(); + SchedulerApplication application = applications.get( applicationAttemptId.getApplicationId()); String user = application.getUser(); @@ -566,9 +555,7 @@ protected void addApplicationAttempt( new RMAppAttemptEvent(applicationAttemptId, RMAppAttemptEventType.ATTEMPT_ADDED)); } - } finally { - writeLock.unlock(); - } + } /** @@ -631,11 +618,10 @@ private void removeApplication(ApplicationId applicationId, } } - private void removeApplicationAttempt( + private synchronized void removeApplicationAttempt( ApplicationAttemptId applicationAttemptId, RMAppAttemptState rmAppAttemptFinalState, boolean keepContainers) { - try { - writeLock.lock(); + LOG.info("Application " + applicationAttemptId + " is done. finalState=" + rmAppAttemptFinalState); FSAppAttempt attempt = getApplicationAttempt(applicationAttemptId); @@ -689,20 +675,16 @@ private void removeApplicationAttempt( } else{ maxRunningEnforcer.untrackNonRunnableApp(attempt); } - } finally { - writeLock.unlock(); - } } /** * Clean up a completed container. */ @Override - protected void completedContainerInternal( + protected synchronized void completedContainerInternal( RMContainer rmContainer, ContainerStatus containerStatus, RMContainerEventType event) { - try { - writeLock.lock(); + Container container = rmContainer.getContainer(); // Get the application for the finished container @@ -742,15 +724,12 @@ protected void completedContainerInternal( + " released container " + container.getId() + " on node: " + (node == null ? nodeID : node) + " with event: " + event); } - } finally { - writeLock.unlock(); - } + } - private void addNode(List containerReports, + private synchronized void addNode(List containerReports, RMNode node) { - try { - writeLock.lock(); + FSSchedulerNode schedulerNode = new FSSchedulerNode(node, usePortForNodeName); nodeTracker.addNode(schedulerNode); @@ -765,14 +744,10 @@ private void addNode(List containerReports, recoverContainersOnNode(containerReports, node); updateRootQueueMetrics(); - } finally { - writeLock.unlock(); - } } - private void removeNode(RMNode rmNode) { - try { - writeLock.lock(); + private synchronized void removeNode(RMNode rmNode) { + NodeId nodeId = rmNode.getNodeID(); FSSchedulerNode node = nodeTracker.getNode(nodeId); if (node == null) { @@ -806,9 +781,6 @@ private void removeNode(RMNode rmNode) { LOG.info("Removed node " + rmNode.getNodeAddress() + " cluster capacity: " + clusterResource); - } finally { - writeLock.unlock(); - } } @Override @@ -869,9 +841,7 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, // Release containers releaseContainers(release, application); - ReentrantReadWriteLock.WriteLock lock = application.getWriteLock(); - lock.lock(); - try { + synchronized (this){ if (!ask.isEmpty()) { if (LOG.isDebugEnabled()) { LOG.debug( @@ -886,8 +856,6 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, // TODO, handle SchedulingRequest application.showRequests(); } - } finally { - lock.unlock(); } Set preemptionContainerIds = @@ -926,9 +894,8 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, } @Override - protected void nodeUpdate(RMNode nm) { - try { - writeLock.lock(); + protected synchronized void nodeUpdate(RMNode nm) { + long start = getClock().getTime(); super.nodeUpdate(nm); @@ -937,9 +904,6 @@ protected void nodeUpdate(RMNode nm) { long duration = getClock().getTime() - start; fsOpDurations.addNodeUpdateDuration(duration); - } finally { - writeLock.unlock(); - } } @Deprecated @@ -1027,9 +991,7 @@ static void assignPreemptedContainers(FSSchedulerNode node) { } @VisibleForTesting - void attemptScheduling(FSSchedulerNode node) { - try { - writeLock.lock(); + synchronized void attemptScheduling(FSSchedulerNode node) { if (rmContext.isWorkPreservingRecoveryEnabled() && !rmContext .isSchedulerReadyForAllocatingContainers()) { return; @@ -1080,9 +1042,6 @@ void attemptScheduling(FSSchedulerNode node) { } } updateRootQueueMetrics(); - } finally { - writeLock.unlock(); - } } public FSAppAttempt getSchedulerApp(ApplicationAttemptId appAttemptId) { @@ -1236,11 +1195,9 @@ public void handle(SchedulerEvent event) { } } - private String resolveReservationQueueName(String queueName, + private synchronized String resolveReservationQueueName(String queueName, ApplicationId applicationId, ReservationId reservationID, boolean isRecovering) { - try { - readLock.lock(); FSQueue queue = queueMgr.getQueue(queueName); if ((queue == null) || !allocConf.isReservable(queue.getQueueName())) { return queueName; @@ -1281,9 +1238,6 @@ private String resolveReservationQueueName(String queueName, queueName = getDefaultQueueForPlanQueue(queueName); } return queueName; - } finally { - readLock.unlock(); - } } @@ -1304,8 +1258,7 @@ public void setRMContext(RMContext rmContext) { @SuppressWarnings("deprecation") private void initScheduler(Configuration conf) throws IOException { - try { - writeLock.lock(); + synchronized (this){ this.conf = new FairSchedulerConfiguration(conf); validateConf(this.conf); authorizer = YarnAuthorizationProvider.getInstance(conf); @@ -1364,8 +1317,6 @@ private void initScheduler(Configuration conf) throws IOException { if (this.conf.getPreemptionEnabled()) { createPreemptionThread(); } - } finally { - writeLock.unlock(); } allocsLoader.init(conf); @@ -1395,9 +1346,8 @@ private void updateReservationThreshold() { reservationThreshold = newThreshold; } - private void startSchedulerThreads() { - try { - writeLock.lock(); + private synchronized void startSchedulerThreads() { + Preconditions.checkNotNull(allocsLoader, "allocsLoader is null"); if (continuousSchedulingEnabled) { Preconditions.checkNotNull(schedulingThread, @@ -1408,9 +1358,6 @@ private void startSchedulerThreads() { preemptionThread.start(); } allocsLoader.start(); - } finally { - writeLock.unlock(); - } } @Override @@ -1430,9 +1377,8 @@ public void serviceStart() throws Exception { @SuppressWarnings("deprecation") @Override - public void serviceStop() throws Exception { - try { - writeLock.lock(); + public synchronized void serviceStop() throws Exception { + if (continuousSchedulingEnabled) { if (schedulingThread != null) { schedulingThread.interrupt(); @@ -1446,10 +1392,6 @@ public void serviceStop() throws Exception { if (allocsLoader != null) { allocsLoader.stop(); } - } finally { - writeLock.unlock(); - } - super.serviceStop(); } @@ -1492,10 +1434,9 @@ public int getNumClusterNodes() { } @Override - public boolean checkAccess(UserGroupInformation callerUGI, + public synchronized boolean checkAccess(UserGroupInformation callerUGI, QueueACL acl, String queueName) { - try { - readLock.lock(); + FSQueue queue = getQueueManager().getQueue(queueName); if (queue == null) { if (LOG.isDebugEnabled()) { @@ -1505,9 +1446,7 @@ public boolean checkAccess(UserGroupInformation callerUGI, return false; } return queue.hasAccess(acl, callerUGI); - } finally { - readLock.unlock(); - } + } public AllocationConfiguration getAllocationConfiguration() { @@ -1518,13 +1457,11 @@ public AllocationConfiguration getAllocationConfiguration() { AllocationFileLoaderService.Listener { @Override - public void onReload(AllocationConfiguration queueInfo) + public synchronized void onReload(AllocationConfiguration queueInfo) throws IOException { // Commit the reload; also create any queue defined in the alloc file // if it does not already exist, so it can be displayed on the web UI. - writeLock.lock(); - try { if (queueInfo == null) { authorizer.setPermission(allocsLoader.getDefaultPermissions(), UserGroupInformation.getCurrentUser()); @@ -1536,9 +1473,6 @@ public void onReload(AllocationConfiguration queueInfo) applyChildDefaults(); maxRunningEnforcer.updateRunnabilityOnReload(); } - } finally { - writeLock.unlock(); - } } } @@ -1598,10 +1532,8 @@ private void applyChildDefaults() { } @Override - public String moveApplication(ApplicationId appId, + public synchronized String moveApplication(ApplicationId appId, String queueName) throws YarnException { - try { - writeLock.lock(); SchedulerApplication app = applications.get(appId); if (app == null) { throw new YarnException("App to be moved " + appId + " not found."); @@ -1609,8 +1541,7 @@ public String moveApplication(ApplicationId appId, FSAppAttempt attempt = (FSAppAttempt) app.getCurrentAppAttempt(); // To serialize with FairScheduler#allocate, synchronize on app attempt - try { - attempt.getWriteLock().lock(); + synchronized (this){ FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue(); // Check if the attempt is already stopped: don't move stopped app // attempt. The attempt has already been removed from all queues. @@ -1635,19 +1566,12 @@ public String moveApplication(ApplicationId appId, executeMove(app, attempt, oldQueue, targetQueue); return targetQueue.getQueueName(); - } finally { - attempt.getWriteLock().unlock(); } - } finally { - writeLock.unlock(); - } } @Override - public void preValidateMoveApplication(ApplicationId appId, String newQueue) + public synchronized void preValidateMoveApplication(ApplicationId appId, String newQueue) throws YarnException { - try { - writeLock.lock(); SchedulerApplication app = applications.get(appId); if (app == null) { throw new YarnException("App to be moved " + appId + " not found."); @@ -1656,8 +1580,7 @@ public void preValidateMoveApplication(ApplicationId appId, String newQueue) FSAppAttempt attempt = app.getCurrentAppAttempt(); // To serialize with FairScheduler#allocate, synchronize on app attempt - try { - attempt.getWriteLock().lock(); + synchronized (this){ FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue(); String destQueueName = handleMoveToPlanQueue(newQueue); FSLeafQueue targetQueue = queueMgr.getLeafQueue(destQueueName, false); @@ -1669,12 +1592,7 @@ public void preValidateMoveApplication(ApplicationId appId, String newQueue) if (oldQueue.isRunnableApp(attempt)) { verifyMoveDoesNotViolateConstraints(attempt, oldQueue, targetQueue); } - } finally { - attempt.getWriteLock().unlock(); } - } finally { - writeLock.unlock(); - } } private void verifyMoveDoesNotViolateConstraints(FSAppAttempt app, @@ -1776,17 +1694,14 @@ FSQueue findLowestCommonAncestorQueue(FSQueue queue1, FSQueue queue2) { * Process resource update on a node and update Queue. */ @Override - public void updateNodeResource(RMNode nm, + public synchronized void updateNodeResource(RMNode nm, ResourceOption resourceOption) { - try { - writeLock.lock(); + super.updateNodeResource(nm, resourceOption); updateRootQueueMetrics(); queueMgr.getRootQueue().setSteadyFairShare(getClusterResource()); queueMgr.getRootQueue().recomputeSteadyShares(); - } finally { - writeLock.unlock(); - } + } /** {@inheritDoc} */ @@ -1857,9 +1772,6 @@ long getNMHeartbeatInterval() { return nmHeartbeatInterval; } - ReadLock getSchedulerReadLock() { - return this.readLock; - } @Override public long checkAndGetApplicationLifetime(String queueName, long lifetime) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java index 169b98a..b489ec7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java @@ -51,10 +51,8 @@ super(appAttemptId, user, queue, activeUsersManager, rmContext); } - public RMContainer allocate(NodeType type, FiCaSchedulerNode node, + public synchronized RMContainer allocate(NodeType type, FiCaSchedulerNode node, SchedulerRequestKey schedulerKey, Container container) { - try { - writeLock.lock(); if (isStopped) { return null; @@ -104,8 +102,5 @@ public RMContainer allocate(NodeType type, FiCaSchedulerNode node, getApplicationId(), containerId, container.getResource()); return rmContainer; - } finally { - writeLock.unlock(); - } } } -- 2.23.0.windows.1