From d17e2f36adc6d16bf3fe5bea4b63a8a6ee39014d Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Thu, 30 Jan 2020 23:04:20 +0530 Subject: [PATCH] YARN-10113. Fix RM failing to start when SystemServiceManagerImpl is configured --- .../service/client/SystemServiceManagerImpl.java | 12 ++++- .../client/TestSystemServiceManagerImpl.java | 51 ++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java index db11f40..3b6fd27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java @@ -113,7 +113,17 @@ protected void serviceInit(Configuration conf) throws Exception { systemServiceDir = new Path(dirPath); LOG.info("System Service Directory is configured to {}", systemServiceDir); - fs = systemServiceDir.getFileSystem(conf); + Configuration serviceConf = new Configuration(conf); + String scheme = systemServiceDir.toUri().getScheme(); + if (scheme == null) { + scheme = FileSystem.getDefaultUri(serviceConf).getScheme(); + } + if (scheme != null) { + String disableCacheName = String.format("fs.%s.impl.disable.cache", + scheme); + serviceConf.setBoolean(disableCacheName, true); + } + fs = systemServiceDir.getFileSystem(serviceConf); this.loginUGI = UserGroupInformation.isSecurityEnabled() ? UserGroupInformation.getLoginUser() : UserGroupInformation.getCurrentUser(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java index 892f2b4..90a9f44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java @@ -18,6 +18,10 @@ package org.apache.hadoop.yarn.service.client; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -40,6 +44,8 @@ import java.util.Map; import java.util.Set; +import static org.junit.Assert.fail; + /** * Test class for system service manager. */ @@ -80,6 +86,51 @@ public void tearDown() { } @Test + public void testFileSystemClose() throws Exception { + MiniDFSCluster hdfsCluster = null; + FileSystem fs = null; + Path path = new Path("/tmp/servicedir"); + + HdfsConfiguration hdfsConfig = new HdfsConfiguration(); + hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig) + .numDataNodes(1).build(); + + fs = hdfsCluster.getFileSystem(); + if (!fs.exists(path)) { + fs.mkdirs(path); + } + + SystemServiceManagerImpl serviceManager = new SystemServiceManagerImpl(); + + hdfsConfig.set(YarnServiceConf.YARN_SERVICES_SYSTEM_SERVICE_DIRECTORY, + path.toString()); + serviceManager.init(hdfsConfig); + + // Close the FileSystem object and validate + fs.close(); + + try { + serviceManager.start(); + } catch (Exception e) { + if (e.getMessage().contains("Filesystem closed")) { + fail("SystemServiceManagerImpl failed to handle " + + "FileSystem close"); + } else { + fail("Should not get any exceptions"); + } + } finally { + serviceManager.stop(); + fs = hdfsCluster.getFileSystem(); + if (fs.exists(path)) { + fs.delete(path, true); + } + if (hdfsCluster != null) { + hdfsCluster.shutdown(); + } + } + } + + public void testSystemServiceSubmission() throws Exception { systemService.start(); -- 2.7.4 (Apple Git-66)