diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 1d04710..4cbd457 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -802,8 +802,7 @@ private void removeApplicationAttempt( RMAppAttemptState rmAppAttemptFinalState, boolean keepContainers) { try { writeLock.lock(); - LOG.info( - "Application " + applicationAttemptId + " is done." + " finalState=" + LOG.info("Application " + applicationAttemptId + " is done. finalState=" + rmAppAttemptFinalState); FSAppAttempt attempt = getApplicationAttempt(applicationAttemptId); @@ -813,6 +812,13 @@ private void removeApplicationAttempt( return; } + // Check if the attempt is already stopped and don't stop it twice. + if (attempt.isStopped()) { + LOG.info("Application " + applicationAttemptId + " has already been " + + "stopped!"); + return; + } + // Release all the running containers for (RMContainer rmContainer : attempt.getLiveContainers()) { if (keepContainers && rmContainer.getState().equals( @@ -1648,6 +1654,12 @@ public String moveApplication(ApplicationId appId, try { attempt.getWriteLock().lock(); FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue(); + // Check if the attempt is already stopped: don't move stopped app + // attempt. The attempt has already been removed from all queues. + if (attempt.isStopped()) { + LOG.info("Application " + appId + " is stopped and can't be moved!"); + return oldQueue.getQueueName(); + } String destQueueName = handleMoveToPlanQueue(queueName); FSLeafQueue targetQueue = queueMgr.getLeafQueue(destQueueName, false); if (targetQueue == null) { @@ -1710,16 +1722,23 @@ private void verifyMoveDoesNotViolateConstraints(FSAppAttempt app, * operations will be atomic. */ private void executeMove(SchedulerApplication app, - FSAppAttempt attempt, FSLeafQueue oldQueue, FSLeafQueue newQueue) { - boolean wasRunnable = oldQueue.removeApp(attempt); + FSAppAttempt attempt, FSLeafQueue oldQueue, FSLeafQueue newQueue) + throws YarnException { + // Check current runs state. Do not remove the attempt from the queue until + // after the check has been performed otherwise it could remove the app + // from a queue without moving it to a new queue. + boolean wasRunnable = oldQueue.isRunnableApp(attempt); // if app was not runnable before, it may be runnable now boolean nowRunnable = maxRunningEnforcer.canAppBeRunnable(newQueue, attempt); if (wasRunnable && !nowRunnable) { - throw new IllegalStateException("Should have already verified that app " + throw new YarnException("Should have already verified that app " + attempt.getApplicationId() + " would be runnable in new queue"); } - + + // Now it is safe to remove from the queue. + oldQueue.removeApp(attempt); + if (wasRunnable) { maxRunningEnforcer.untrackRunnableApp(attempt); } else if (nowRunnable) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 21d22c3..119b887 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -4415,6 +4415,59 @@ public void testLowestCommonAncestorDeeperHierarchy() throws Exception { } @Test + public void testDoubleRemoval() throws Exception { + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + ApplicationAttemptId attemptId = createAppAttemptId(1, 1); + AppAddedSchedulerEvent appAddedEvent = + new AppAddedSchedulerEvent(attemptId.getApplicationId(), "default", + "user1"); + scheduler.handle(appAddedEvent); + AppAttemptAddedSchedulerEvent attemptAddedEvent = + new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false); + scheduler.handle(attemptAddedEvent); + + AppAttemptRemovedSchedulerEvent attemptRemovedEvent = + new AppAttemptRemovedSchedulerEvent(createAppAttemptId(1, 1), + RMAppAttemptState.FINISHED, false); + + // Now remove the app attempt + scheduler.handle(attemptRemovedEvent); + + // Now remove the app attempt again, since it is stopped nothing happens + scheduler.handle(attemptRemovedEvent); + } + + @Test + public void testMoveAfterRemoval() throws Exception { + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + ApplicationAttemptId attemptId = createAppAttemptId(1, 1); + AppAddedSchedulerEvent appAddedEvent = + new AppAddedSchedulerEvent(attemptId.getApplicationId(), "test1", + "user1"); + scheduler.handle(appAddedEvent); + AppAttemptAddedSchedulerEvent attemptAddedEvent = + new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false); + scheduler.handle(attemptAddedEvent); + + AppAttemptRemovedSchedulerEvent attemptRemovedEvent = + new AppAttemptRemovedSchedulerEvent(createAppAttemptId(1, 1), + RMAppAttemptState.FINISHED, false); + + // Remove the app attempt + scheduler.handle(attemptRemovedEvent); + + // Now move the app: not using an event since there is none + // in the scheduler + scheduler.moveApplication(attemptId.getApplicationId(), "default"); + } + + @Test public void testPerfMetricsInited() { scheduler.init(conf); scheduler.start();