diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index fddee46..38112c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -35,7 +35,9 @@ function hadoop_usage echo " proxyserver run the web app proxy server" echo " queue prints queue information" echo " resourcemanager run the ResourceManager" - echo " resourcemanager -format-state-store deletes the RMStateStore" + echo " Use -format-state-store for deleting the RMStateStore." + echo " Use -remove-application-from-state-store for removing" + echo " application from RMStateStore." echo " rmadmin admin tools" echo " scmadmin SharedCacheManager admin tools" echo " sharedcachemanager run the SharedCacheManager daemon" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 8bd8e21..b73b6a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -93,6 +93,7 @@ import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; import org.apache.hadoop.yarn.server.webproxy.WebAppProxy; import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; import org.apache.hadoop.yarn.webapp.WebApps.Builder; @@ -1199,6 +1200,9 @@ public static void main(String argv[]) { // If -format-state-store, then delete RMStateStore; else startup normally if (argv.length == 1 && argv[0].equals("-format-state-store")) { deleteRMStateStore(conf); + } else if (argv.length == 2 + && argv[0].equals("-remove-application-from-stare-store")) { + removeApplication(conf, argv[1]); } else { ResourceManager resourceManager = new ResourceManager(); ShutdownHookManager.get().addShutdownHook( @@ -1275,4 +1279,17 @@ private static void deleteRMStateStore(Configuration conf) throws Exception { rmStore.stop(); } } + + private static void removeApplication(Configuration conf, String applicationId) + throws Exception { + RMStateStore rmStore = RMStateStoreFactory.getStore(conf); + rmStore.init(conf); + rmStore.start(); + try { + ApplicationId removeAppId = ConverterUtils.toApplicationId(applicationId); + rmStore.removeApplication(removeAppId); + } finally { + rmStore.stop(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index 7652a07..68d26bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -559,6 +559,15 @@ public synchronized void deleteStore() throws Exception { } } + @Override + public synchronized void removeApplication(ApplicationId removeAppId) + throws Exception { + Path nodeRemovePath = getAppDir(rmAppRoot, removeAppId); + if (existsWithRetries(nodeRemovePath)) { + deleteFileWithRetries(nodeRemovePath); + } + } + private Path getAppDir(Path root, ApplicationId appId) { return getNodePath(root, appId.toString()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java index 2c92714..46a3459 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java @@ -659,6 +659,18 @@ public void deleteStore() throws IOException { fs.delete(root, true); } + @Override + public synchronized void removeApplication(ApplicationId removeAppId) + throws IOException { + String appKey = getApplicationNodeKey(removeAppId); + LOG.info("Removing state for app " + removeAppId); + try { + db.delete(bytes(appKey)); + } catch (DBException e) { + throw new IOException(e); + } + } + @VisibleForTesting int getNumEntriesInDatabase() throws IOException { int numEntries = 0; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java index 8cd776e..609f403 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java @@ -251,4 +251,8 @@ public synchronized void storeOrUpdateAMRMTokenSecretManagerState( public void deleteStore() throws Exception { } + @Override + public void removeApplication(ApplicationId removeAppId) throws Exception { + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java index d2c1e9d..92c07cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java @@ -150,4 +150,9 @@ public void deleteStore() throws Exception { // Do nothing } + @Override + public void removeApplication(ApplicationId removeAppId) throws Exception { + // Do nothing + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index bccde53..cc4edd7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -907,6 +907,15 @@ public void handle(RMStateStoreEvent event) { */ public abstract void deleteStore() throws Exception; + /** + * Derived classes must implement this method to remove application from the + * state store + * + * @throws Exception + */ + public abstract void removeApplication(ApplicationId removeAppId) + throws Exception; + public void setResourceManager(ResourceManager rm) { this.resourceManager = rm; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index 614ef15..842fc75 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -823,6 +823,15 @@ public synchronized void deleteStore() throws Exception { } } + @Override + public synchronized void removeApplication(ApplicationId removeAppId) + throws Exception { + String appIdRemovePath = getNodePath(rmAppRoot, removeAppId.toString()); + if (existsWithRetries(appIdRemovePath, true) != null) { + deleteWithRetries(appIdRemovePath, true); + } + } + // ZK related code /** * Watcher implementation which forward events to the ZKRMStateStore This diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index 5b53a02..3bc0709 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -591,6 +591,21 @@ public void testDeleteStore(RMStateStoreHelper stateStoreHelper) } } + public void testRemoveApplication(RMStateStoreHelper stateStoreHelper) + throws Exception { + RMStateStore store = stateStoreHelper.getRMStateStore(); + int noOfApps = 2; + ArrayList appList = + createAndStoreApps(stateStoreHelper, store, noOfApps); + + RMApp rmApp1 = appList.get(0); + store.removeApplication(rmApp1.getApplicationId()); + Assert.assertFalse(stateStoreHelper.appExists(rmApp1)); + + RMApp rmApp2 = appList.get(1); + Assert.assertTrue(stateStoreHelper.appExists(rmApp2)); + } + protected void modifyAppState() throws Exception { } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java index c5f5578..b1e7a0b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java @@ -171,6 +171,7 @@ public void testFSRMStateStore() throws Exception { testEpoch(fsTester); testAppDeletion(fsTester); testDeleteStore(fsTester); + testRemoveApplication(fsTester); testAMRMTokenSecretManagerStateStore(fsTester); } finally { cluster.shutdown(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java index ae885d2..17cffa0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java @@ -91,6 +91,12 @@ public void testDeleteStore() throws Exception { } @Test(timeout = 60000) + public void testRemoveApplication() throws Exception { + LeveldbStateStoreTester tester = new LeveldbStateStoreTester(); + testRemoveApplication(tester); + } + + @Test(timeout = 60000) public void testAMTokens() throws Exception { LeveldbStateStoreTester tester = new LeveldbStateStoreTester(); testAMRMTokenSecretManagerStateStore(tester); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index 608adbe..6b09d39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -146,6 +146,7 @@ public void testZKRMStateStoreRealZK() throws Exception { testEpoch(zkTester); testAppDeletion(zkTester); testDeleteStore(zkTester); + testRemoveApplication(zkTester); testAMRMTokenSecretManagerStateStore(zkTester); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index b4bed64..c5f5e22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -197,6 +197,7 @@ Usage: `yarn resourcemanager [-format-state-store]` | COMMAND\_OPTIONS | Description | |:---- |:---- | | -format-state-store | Formats the RMStateStore. This will clear the RMStateStore and is useful if past applications are no longer needed. This should be run only when the ResourceManager is not running. | +| -remove-application-from-state-store | Remove the application from RMStateStore. This should be run only when the ResourceManager is not running. | Start the ResourceManager -- 1.9.2.msysgit.0