diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index fbcac76..8fa971f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -633,8 +633,7 @@ private void removeApplicationAttempt( RMAppAttemptState rmAppAttemptFinalState, boolean keepContainers) { try { writeLock.lock(); - LOG.info( - "Application " + applicationAttemptId + " is done." + " finalState=" + LOG.info("Application " + applicationAttemptId + " is done. finalState=" + rmAppAttemptFinalState); FSAppAttempt attempt = getApplicationAttempt(applicationAttemptId); @@ -644,6 +643,13 @@ private void removeApplicationAttempt( return; } + // Check if the attempt is already stopped and don't stop it twice. + if (attempt.isStopped()) { + LOG.info("Application " + applicationAttemptId + " has already been " + + "stopped!"); + return; + } + // Release all the running containers for (RMContainer rmContainer : attempt.getLiveContainers()) { if (keepContainers && rmContainer.getState().equals( @@ -1521,6 +1527,13 @@ public String moveApplication(ApplicationId appId, try { attempt.getWriteLock().lock(); FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue(); + // Check if the attempt is already stopped: don't move stopped app + // attempt. The attempt has already been removed from all queues. + if (attempt.isStopped()) { + LOG.info("Application " + appId + " is stopped and can't be moved!"); + throw new YarnException("Application " + appId + + " is stopped and can't be moved!"); + } String destQueueName = handleMoveToPlanQueue(queueName); FSLeafQueue targetQueue = queueMgr.getLeafQueue(destQueueName, false); if (targetQueue == null) { @@ -1583,16 +1596,23 @@ private void verifyMoveDoesNotViolateConstraints(FSAppAttempt app, * operations will be atomic. */ private void executeMove(SchedulerApplication app, - FSAppAttempt attempt, FSLeafQueue oldQueue, FSLeafQueue newQueue) { - boolean wasRunnable = oldQueue.removeApp(attempt); + FSAppAttempt attempt, FSLeafQueue oldQueue, FSLeafQueue newQueue) + throws YarnException { + // Check current runs state. Do not remove the attempt from the queue until + // after the check has been performed otherwise it could remove the app + // from a queue without moving it to a new queue. + boolean wasRunnable = oldQueue.isRunnableApp(attempt); // if app was not runnable before, it may be runnable now boolean nowRunnable = maxRunningEnforcer.canAppBeRunnable(newQueue, attempt); if (wasRunnable && !nowRunnable) { - throw new IllegalStateException("Should have already verified that app " + throw new YarnException("Should have already verified that app " + attempt.getApplicationId() + " would be runnable in new queue"); } - + + // Now it is safe to remove from the queue. + oldQueue.removeApp(attempt); + if (wasRunnable) { maxRunningEnforcer.untrackRunnableApp(attempt); } else if (nowRunnable) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 55f8849..5aa1e2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -4552,6 +4552,95 @@ public void testLowestCommonAncestorDeeperHierarchy() throws Exception { } @Test + public void testDoubleRemoval() throws Exception { + String testUser = "user1"; // convenience var + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + ApplicationAttemptId attemptId = createAppAttemptId(1, 1); + // The placement rule will add the app to the user based queue but the + // passed in queue must exist. + AppAddedSchedulerEvent appAddedEvent = + new AppAddedSchedulerEvent(attemptId.getApplicationId(), testUser, + testUser); + scheduler.handle(appAddedEvent); + AppAttemptAddedSchedulerEvent attemptAddedEvent = + new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false); + scheduler.handle(attemptAddedEvent); + + // Get a handle on the attempt. + FSAppAttempt attempt = scheduler.getSchedulerApp(attemptId); + + AppAttemptRemovedSchedulerEvent attemptRemovedEvent = + new AppAttemptRemovedSchedulerEvent(createAppAttemptId(1, 1), + RMAppAttemptState.FINISHED, false); + + // Make sure the app attempt is in the queue. + List attemptList = + scheduler.getAppsInQueue(testUser); + assertNotNull("Queue missing", attemptList); + assertTrue("Attempt should be in the queue", + attemptList.contains(attemptId)); + assertFalse("Attempt is stopped", attempt.isStopped()); + + // Now remove the app attempt + scheduler.handle(attemptRemovedEvent); + // The attempt is not in the queue, and stopped + attemptList = scheduler.getAppsInQueue(testUser); + assertFalse("Attempt should not be in the queue", + attemptList.contains(attemptId)); + assertTrue("Attempt should have been stopped", attempt.isStopped()); + + // Now remove the app attempt again, since it is stopped nothing happens. + scheduler.handle(attemptRemovedEvent); + // The attempt should still show the original queue info. + assertTrue("Attempt queue has changed", + attempt.getQueue().getName().endsWith(testUser)); + } + + @Test (expected = YarnException.class) + public void testMoveAfterRemoval() throws Exception { + String testUser = "user1"; // convenience var + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + ApplicationAttemptId attemptId = createAppAttemptId(1, 1); + AppAddedSchedulerEvent appAddedEvent = + new AppAddedSchedulerEvent(attemptId.getApplicationId(), testUser, + testUser); + scheduler.handle(appAddedEvent); + AppAttemptAddedSchedulerEvent attemptAddedEvent = + new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false); + scheduler.handle(attemptAddedEvent); + + // Get a handle on the attempt. + FSAppAttempt attempt = scheduler.getSchedulerApp(attemptId); + + AppAttemptRemovedSchedulerEvent attemptRemovedEvent = + new AppAttemptRemovedSchedulerEvent(createAppAttemptId(1, 1), + RMAppAttemptState.FINISHED, false); + + // Remove the app attempt + scheduler.handle(attemptRemovedEvent); + // Make sure the app attempt is not in the queue and stopped. + List attemptList = + scheduler.getAppsInQueue(testUser); + assertNotNull("Queue missing", attemptList); + assertFalse("Attempt should not be in the queue", + attemptList.contains(attemptId)); + assertTrue("Attempt should have been stopped", attempt.isStopped()); + // The attempt should still show the original queue info. + assertTrue("Attempt queue has changed", + attempt.getQueue().getName().endsWith(testUser)); + + // Now move the app: not using an event since there is none + // in the scheduler. This should throw. + scheduler.moveApplication(attemptId.getApplicationId(), "default"); + } + + @Test public void testPerfMetricsInited() { scheduler.init(conf); scheduler.start();