diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 05c6cbf..aad119a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -508,6 +508,12 @@ private static void addDeprecatedKeys() { public static final String DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC = "2000, 500"; + public static final String + FS_RM_STATE_STORE_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES = RM_PREFIX + + "fs.state-store.block-write-locateFollowingBlock-retries"; + public static final int + DEFAULT_FS_RM_STATE_STORE_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES = 8; + public static final String RM_LEVELDB_STORE_PATH = RM_PREFIX + "leveldb-state-store.path"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index a7958a5..22e9a46 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -420,6 +420,14 @@ + customize hdfs client retries of locateFollowingBlock and + completeFile for FileSystemRMStateStore. + + yarn.resourcemanager.fs.state-store.block-write-locateFollowingBlock-retries + 8 + + + Local path where the RM state will be stored when using org.apache.hadoop.yarn.server.resourcemanager.recovery.LeveldbRMStateStore as the value for yarn.resourcemanager.store.class diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index 6e830a0..099a663 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -119,6 +119,12 @@ protected synchronized void startInternal() throws Exception { conf.get(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC, YarnConfiguration.DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC); conf.set("dfs.client.retry.policy.spec", retryPolicy); + int blockWriteLocateFollowingRetry = conf.getInt(YarnConfiguration. + FS_RM_STATE_STORE_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES, + YarnConfiguration. + DEFAULT_FS_RM_STATE_STORE_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES); + conf.setInt("dfs.client.block.write.locateFollowingBlock.retries", + blockWriteLocateFollowingRetry); fs = fsWorkingPath.getFileSystem(conf); fs.mkdirs(rmDTSecretManagerRoot); @@ -641,4 +647,8 @@ public synchronized void storeOrUpdateAMRMTokenSecretManagerState( } } + @VisibleForTesting + FileSystem getFileSystem() { + return fs; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java index d0d19e3..1ee0ca3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java @@ -100,7 +100,12 @@ public RMStateStore getRMStateStore() throws Exception { workingDirPathURI.toString()); conf.set(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC, "100,6000"); + conf.setInt(YarnConfiguration. + FS_RM_STATE_STORE_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES, 10); + conf.setBoolean("fs.hdfs.impl.disable.cache", true); this.store = new TestFileSystemRMStore(conf); + Assert.assertEquals(10, store.getFileSystem().getConf().getInt( + "dfs.client.block.write.locateFollowingBlock.retries", 0)); return store; }