diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index fa6fec4..d8bd077 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -69,8 +69,10 @@ function print_usage(){ echo " CLASSNAME run the class named CLASSNAME" echo " or" echo " where COMMAND is one of:" - echo " resourcemanager -format-state-store deletes the RMStateStore" echo " resourcemanager run the ResourceManager" + echo " Use -format-state-store for deleting the RMStateStore." + echo " Use -remove-application-from-state-store for " + echo " removing application from RMStateStore." echo " nodemanager run a nodemanager on each slave" echo " timelineserver run the timeline server" echo " rmadmin admin tools" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 8bd8e21..130cfd4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -93,6 +93,7 @@ import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; import org.apache.hadoop.yarn.server.webproxy.WebAppProxy; import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; import org.apache.hadoop.yarn.webapp.WebApps.Builder; @@ -100,6 +101,7 @@ import java.io.IOException; import java.io.InputStream; +import java.io.PrintStream; import java.net.InetSocketAddress; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; @@ -1197,8 +1199,15 @@ public static void main(String argv[]) { GenericOptionsParser hParser = new GenericOptionsParser(conf, argv); argv = hParser.getRemainingArgs(); // If -format-state-store, then delete RMStateStore; else startup normally - if (argv.length == 1 && argv[0].equals("-format-state-store")) { - deleteRMStateStore(conf); + if (argv.length >= 1) { + if (argv[0].equals("-format-state-store")) { + deleteRMStateStore(conf); + } else if (argv[0].equals("-remove-application-from-state-store") + && argv.length == 2) { + removeApplication(conf, argv[1]); + } else { + printUsage(System.err); + } } else { ResourceManager resourceManager = new ResourceManager(); ShutdownHookManager.get().addShutdownHook( @@ -1275,4 +1284,25 @@ private static void deleteRMStateStore(Configuration conf) throws Exception { rmStore.stop(); } } + + private static void removeApplication(Configuration conf, String applicationId) + throws Exception { + RMStateStore rmStore = RMStateStoreFactory.getStore(conf); + rmStore.init(conf); + rmStore.start(); + try { + ApplicationId removeAppId = ConverterUtils.toApplicationId(applicationId); + LOG.info("Deleting application " + removeAppId + " from state store"); + rmStore.removeApplication(removeAppId); + LOG.info("Application is deleted from state store"); + } finally { + rmStore.stop(); + } + } + + private static void printUsage(PrintStream out) { + out.println("Usage: java ResourceManager [-format-state-store]"); + out.println(" " + + "[-remove-application-from-state-store ]" + "\n"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index 7652a07..68d26bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -559,6 +559,15 @@ public synchronized void deleteStore() throws Exception { } } + @Override + public synchronized void removeApplication(ApplicationId removeAppId) + throws Exception { + Path nodeRemovePath = getAppDir(rmAppRoot, removeAppId); + if (existsWithRetries(nodeRemovePath)) { + deleteFileWithRetries(nodeRemovePath); + } + } + private Path getAppDir(Path root, ApplicationId appId) { return getNodePath(root, appId.toString()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java index 2c92714..46a3459 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java @@ -659,6 +659,18 @@ public void deleteStore() throws IOException { fs.delete(root, true); } + @Override + public synchronized void removeApplication(ApplicationId removeAppId) + throws IOException { + String appKey = getApplicationNodeKey(removeAppId); + LOG.info("Removing state for app " + removeAppId); + try { + db.delete(bytes(appKey)); + } catch (DBException e) { + throw new IOException(e); + } + } + @VisibleForTesting int getNumEntriesInDatabase() throws IOException { int numEntries = 0; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java index 8cd776e..609f403 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java @@ -251,4 +251,8 @@ public synchronized void storeOrUpdateAMRMTokenSecretManagerState( public void deleteStore() throws Exception { } + @Override + public void removeApplication(ApplicationId removeAppId) throws Exception { + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java index d2c1e9d..92c07cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java @@ -150,4 +150,9 @@ public void deleteStore() throws Exception { // Do nothing } + @Override + public void removeApplication(ApplicationId removeAppId) throws Exception { + // Do nothing + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index bccde53..cc4edd7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -907,6 +907,15 @@ public void handle(RMStateStoreEvent event) { */ public abstract void deleteStore() throws Exception; + /** + * Derived classes must implement this method to remove application from the + * state store + * + * @throws Exception + */ + public abstract void removeApplication(ApplicationId removeAppId) + throws Exception; + public void setResourceManager(ResourceManager rm) { this.resourceManager = rm; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index e8891a2..9da6400 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -823,6 +823,15 @@ public synchronized void deleteStore() throws Exception { } } + @Override + public synchronized void removeApplication(ApplicationId removeAppId) + throws Exception { + String appIdRemovePath = getNodePath(rmAppRoot, removeAppId.toString()); + if (existsWithRetries(appIdRemovePath, true) != null) { + deleteWithRetries(appIdRemovePath, true); + } + } + // ZK related code /** * Watcher implementation which forward events to the ZKRMStateStore This diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index 5b53a02..3bc0709 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -591,6 +591,21 @@ public void testDeleteStore(RMStateStoreHelper stateStoreHelper) } } + public void testRemoveApplication(RMStateStoreHelper stateStoreHelper) + throws Exception { + RMStateStore store = stateStoreHelper.getRMStateStore(); + int noOfApps = 2; + ArrayList appList = + createAndStoreApps(stateStoreHelper, store, noOfApps); + + RMApp rmApp1 = appList.get(0); + store.removeApplication(rmApp1.getApplicationId()); + Assert.assertFalse(stateStoreHelper.appExists(rmApp1)); + + RMApp rmApp2 = appList.get(1); + Assert.assertTrue(stateStoreHelper.appExists(rmApp2)); + } + protected void modifyAppState() throws Exception { } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java index c5f5578..b1e7a0b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java @@ -171,6 +171,7 @@ public void testFSRMStateStore() throws Exception { testEpoch(fsTester); testAppDeletion(fsTester); testDeleteStore(fsTester); + testRemoveApplication(fsTester); testAMRMTokenSecretManagerStateStore(fsTester); } finally { cluster.shutdown(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java index ae885d2..17cffa0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java @@ -91,6 +91,12 @@ public void testDeleteStore() throws Exception { } @Test(timeout = 60000) + public void testRemoveApplication() throws Exception { + LeveldbStateStoreTester tester = new LeveldbStateStoreTester(); + testRemoveApplication(tester); + } + + @Test(timeout = 60000) public void testAMTokens() throws Exception { LeveldbStateStoreTester tester = new LeveldbStateStoreTester(); testAMRMTokenSecretManagerStateStore(tester); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index 608adbe..6b09d39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -146,6 +146,7 @@ public void testZKRMStateStoreRealZK() throws Exception { testEpoch(zkTester); testAppDeletion(zkTester); testDeleteStore(zkTester); + testRemoveApplication(zkTester); testAMRMTokenSecretManagerStateStore(zkTester); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index db8201a..1169d14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -193,6 +193,7 @@ Usage: `yarn resourcemanager [-format-state-store]` | COMMAND\_OPTIONS | Description | |:---- |:---- | | -format-state-store | Formats the RMStateStore. This will clear the RMStateStore and is useful if past applications are no longer needed. This should be run only when the ResourceManager is not running. | +| -remove-application-from-state-store | Remove the application from RMStateStore. This should be run only when the ResourceManager is not running. | Start the ResourceManager -- 1.9.2.msysgit.0