From 8f00decf914379be242273b9d46663a1610a61d9 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Mon, 7 Oct 2019 23:21:05 +0530 Subject: [PATCH] YARN-9875. Handle FileSystem close in FSSchedulerConfigurationStore. --- .../conf/FSSchedulerConfigurationStore.java | 12 ++++- .../conf/TestFSSchedulerConfigurationStore.java | 51 ++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java index 80053be..b1e41c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java @@ -64,7 +64,7 @@ private Path tempConfigPath; @Override - public void initialize(Configuration conf, Configuration vSchedConf, + public void initialize(Configuration fsConf, Configuration vSchedConf, RMContext rmContext) throws Exception { this.configFilePathFilter = new PathFilter() { @Override @@ -78,6 +78,7 @@ public boolean accept(Path path) { } }; + Configuration conf = new Configuration(fsConf); String schedulerConfPathStr = conf.get( YarnConfiguration.SCHEDULER_CONFIGURATION_FS_PATH); if (schedulerConfPathStr == null || schedulerConfPathStr.isEmpty()) { @@ -86,6 +87,15 @@ public boolean accept(Path path) { + " must be set"); } this.schedulerConfDir = new Path(schedulerConfPathStr); + String scheme = schedulerConfDir.toUri().getScheme(); + if (scheme == null) { + scheme = FileSystem.getDefaultUri(conf).getScheme(); + } + if (scheme != null) { + String disableCacheName = String.format("fs.%s.impl.disable.cache", + scheme); + conf.setBoolean(disableCacheName, true); + } this.fileSystem = this.schedulerConfDir.getFileSystem(conf); this.maxVersion = conf.getInt( YarnConfiguration.SCHEDULER_CONFIGURATION_FS_MAX_VERSION, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java index f3d5e74..dcf7f09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java @@ -28,6 +28,8 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation; import org.junit.After; @@ -139,6 +141,55 @@ public void confirmMutationWithInValid() throws Exception { } @Test + public void testFileSystemClose() throws Exception { + MiniDFSCluster hdfsCluster = null; + FileSystem fs = null; + Path path = new Path("/tmp/confstore"); + try { + HdfsConfiguration hdfsConfig = new HdfsConfiguration(); + hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig) + .numDataNodes(1).build(); + + fs = hdfsCluster.getFileSystem(); + if (!fs.exists(path)) { + fs.mkdirs(path); + } + + FSSchedulerConfigurationStore configurationStore = + new FSSchedulerConfigurationStore(); + hdfsConfig.set(YarnConfiguration.SCHEDULER_CONFIGURATION_FS_PATH, + path.toString()); + configurationStore.initialize(hdfsConfig, hdfsConfig, null); + + // Close the FileSystem object and validate + fs.close(); + + try { + Map updates = new HashMap<>(); + updates.put("testkey", "testvalue"); + LogMutation logMutation = new LogMutation(updates, "test"); + configurationStore.logMutation(logMutation); + configurationStore.confirmMutation(true); + } catch (IOException e) { + if (e.getMessage().contains("Filesystem closed")) { + fail("FSSchedulerConfigurationStore failed to handle " + + "FileSystem close"); + } else { + fail("Should not get any exceptions"); + } + } + } finally { + fs = hdfsCluster.getFileSystem(); + if (fs.exists(path)) { + fs.delete(path, true); + } + if (hdfsCluster != null) { + hdfsCluster.shutdown(); + } + } + } + + @Test public void testFormatConfiguration() throws Exception { assertTrue(testSchedulerConfigurationDir.exists()); Configuration schedulerConf = new Configuration(); -- 2.7.4 (Apple Git-66)