diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationContext.java new file mode 100644 index 0000000..c3ff903 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationContext.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.conf; + +import java.io.InputStream; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + *

ConfigurationContext represents all needed + * information of a given Configuration file.

+ * + *

It includes details such as: + *

+ *

+ * @see ConfigurationProvider#getConfigurationContext(String) + */ +@Private +@Unstable +public class ConfigurationContext { + private String name; + private InputStream inputStream; + private long lastModifiedTime; + + public ConfigurationContext(String name, InputStream inputStream, + long lastModifiedTime) { + this.name = name; + this.inputStream = inputStream; + this.lastModifiedTime = lastModifiedTime; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public InputStream getInputStream() { + return inputStream; + } + + public void setInputStream(InputStream inputStream) { + this.inputStream = inputStream; + } + + public long getLastModifiedTime() { + return lastModifiedTime; + } + + public void setLastModifiedTime(long lastModifiedTime) { + this.lastModifiedTime = lastModifiedTime; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java index b31573d..c173a19 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java @@ -42,15 +42,14 @@ public void close() throws Exception { } /** - * Get the configuration and combine with bootstrapConf - * @param bootstrapConf Configuration + * create a {@link ConfigurationContext} object for the indicated Configuration file * @param name The configuration file name - * @return configuration + * @return configurationContext * @throws YarnException * @throws IOException */ - public abstract Configuration getConfiguration(Configuration bootstrapConf, - String name) throws YarnException, IOException; + public abstract ConfigurationContext getConfigurationContext(String name) + throws YarnException, IOException; /** * Derived classes initialize themselves using this method. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 9612cac..f07a3e3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -45,22 +45,30 @@ "hadoop-policy.xml"; @Private - public static final String YARN_SITE_XML_FILE = "yarn-site.xml"; + public static final String YARN_SITE_CONFIGURATION_FILE = "yarn-site.xml"; + private static final String YARN_DEFAULT_CONFIGURATION_FILE = + "yarn-default.xml"; @Private public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml"; + @Private + public static final List RM_CONFIGURATION_FILES = + Collections.unmodifiableList(Arrays.asList( + CS_CONFIGURATION_FILE, + HADOOP_POLICY_CONFIGURATION_FILE, + YARN_SITE_CONFIGURATION_FILE, + CORE_SITE_CONFIGURATION_FILE)); + @Evolving public static final int APPLICATION_MAX_TAGS = 10; @Evolving public static final int APPLICATION_MAX_TAG_LENGTH = 100; - private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; - static { - Configuration.addDefaultResource(YARN_DEFAULT_XML_FILE); - Configuration.addDefaultResource(YARN_SITE_XML_FILE); + Configuration.addDefaultResource(YARN_DEFAULT_CONFIGURATION_FILE); + Configuration.addDefaultResource(YARN_SITE_CONFIGURATION_FILE); } //Configurations diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java index a57d507..1713798 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java @@ -23,6 +23,9 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; @@ -30,6 +33,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.ha.ClientBaseWithFixes; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.proto.HAServiceProtocolProtos; @@ -41,7 +46,10 @@ import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.MiniYARNCluster; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; import org.apache.hadoop.yarn.server.resourcemanager.AdminService; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer; import org.junit.After; import org.junit.Assert; @@ -63,6 +71,8 @@ private Configuration conf; private MiniYARNCluster cluster; private ApplicationId fakeAppId; + private FileSystem fs; + private Path workingPath; private void setConfForRM(String rmId, String prefix, String value) { @@ -98,6 +108,15 @@ public void setup() throws IOException { conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true); + fs = FileSystem.get(conf); + workingPath = + new Path(new File("target", this.getClass().getSimpleName() + + "-remoteDir").getAbsolutePath()); + conf.set(YarnConfiguration.FS_BASED_RM_CONF_STORE, + workingPath.toString()); + fs.delete(workingPath, true); + fs.mkdirs(workingPath); + cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 1, 1, 1); } @@ -245,6 +264,47 @@ public void testEmbeddedWebAppProxy() throws YarnException, verifyExpectedException(proxyConn.getResponseMessage()); } + @Test + public void testRMFailOverWithFileSystemBasedConfigurationProvider() + throws IOException, YarnException, InterruptedException { + conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); + conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + // upload default configurations + uploadDefaultConfiguration(); + YarnConfiguration yarnConf = new YarnConfiguration(conf); + uploadConfiguration(yarnConf, "yarn-site.xml"); + + cluster.init(conf); + cluster.start(); + getAdminService(0).transitionToActive(req); + assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex()); + + CapacityScheduler cs = + (CapacityScheduler) cluster.getResourceManager(0).getRMContext() + .getScheduler(); + int maxAppsBefore = cs.getConfiguration().getMaximumSystemApplications(); + + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + csConf.set("yarn.scheduler.capacity.maximum-applications", "5000"); + uploadConfiguration(csConf, "capacity-scheduler.xml"); + + cluster.getResourceManager(0).getRMContext().getRMAdminService() + .refreshQueues(RefreshQueuesRequest.newInstance()); + + int maxAppsAfter = cs.getConfiguration().getMaximumSystemApplications(); + Assert.assertEquals(maxAppsAfter, 5000); + Assert.assertTrue(maxAppsAfter != maxAppsBefore); + + explicitFailover(); + verifyConnections(); + int maxAppsAfterFailOver = + ((CapacityScheduler) cluster.getResourceManager(1).getRMContext() + .getScheduler()).getConfiguration().getMaximumSystemApplications(); + Assert.assertEquals(maxAppsAfterFailOver, 5000); + } + private void verifyExpectedException(String exceptionMessage){ assertTrue(exceptionMessage.contains(ApplicationNotFoundException.class .getName())); @@ -252,4 +312,57 @@ private void verifyExpectedException(String exceptionMessage){ .contains("Application with id '" + fakeAppId + "' " + "doesn't exist in RM.")); } + + private String writeConfigurationXML(Configuration conf, String confXMLName) + throws IOException { + DataOutputStream output = null; + try { + final File confFile = new File(tmpDir.toString(), confXMLName); + if (confFile.exists()) { + confFile.delete(); + } + if (!confFile.createNewFile()) { + Assert.fail("Can not create " + confXMLName); + } + output = new DataOutputStream( + new FileOutputStream(confFile)); + conf.writeXml(output); + return confFile.getAbsolutePath(); + } finally { + if (output != null) { + output.close(); + } + } + } + + private void uploadToRemoteFileSystem(Path filePath) + throws IOException { + fs.copyFromLocalFile(filePath, workingPath); + } + + private void uploadConfiguration(Configuration conf, String confFileName) + throws IOException { + String csConfFile = writeConfigurationXML(conf, confFileName); + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(csConfFile)); + } + + private void uploadDefaultConfiguration() throws IOException { + Configuration conf = new Configuration(); + uploadConfiguration(conf, "core-site.xml"); + + YarnConfiguration yarnConf = new YarnConfiguration(); + yarnConf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + uploadConfiguration(yarnConf, "yarn-site.xml"); + + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + uploadConfiguration(csConf, "capacity-scheduler.xml"); + + Configuration hadoopPolicyConf = new Configuration(false); + hadoopPolicyConf + .addResource(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); + uploadConfiguration(hadoopPolicyConf, "hadoop-policy.xml"); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java index 390aace..8bbbbbc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java @@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.conf.ConfigurationContext; import org.apache.hadoop.yarn.conf.ConfigurationProvider; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -41,15 +42,27 @@ private Path configDir; @Override - public synchronized Configuration getConfiguration(Configuration bootstrapConf, - String name) throws IOException, YarnException { - Path configPath = new Path(this.configDir, name); - if (!fs.exists(configPath)) { - throw new YarnException("Can not find Configuration: " + name + " in " - + configDir); + public synchronized ConfigurationContext getConfigurationContext(String name) + throws IOException, YarnException { + if (name == null || name.isEmpty()) { + throw new YarnException( + "Illegal argument! The parameter should not be null or empty"); } - bootstrapConf.addResource(fs.open(configPath)); - return bootstrapConf; + Path filePath; + if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name)) { + filePath = new Path(this.configDir, name); + if (!fs.exists(filePath)) { + throw new YarnException("Can not find Configuration: " + name + " in " + + configDir); + } + } else { + filePath = new Path(name); + if (!fs.exists(filePath)) { + throw new YarnException("Can not find file: " + name); + } + } + return new ConfigurationContext(name, fs.open(filePath), fs.getFileStatus( + filePath).getModificationTime()); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java index 3e69960..2e4ce66 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java @@ -18,21 +18,49 @@ package org.apache.hadoop.yarn; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.net.URL; + import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.ConfigurationContext; import org.apache.hadoop.yarn.conf.ConfigurationProvider; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @Private @Unstable public class LocalConfigurationProvider extends ConfigurationProvider { + private ClassLoader classLoader; + { + classLoader = Thread.currentThread().getContextClassLoader(); + if (classLoader == null) { + classLoader = Configuration.class.getClassLoader(); + } + } @Override - public Configuration getConfiguration(Configuration bootstrapConf, - String name) throws IOException, YarnException { - return bootstrapConf; + public ConfigurationContext getConfigurationContext(String name) + throws IOException, YarnException { + if (name == null || name.isEmpty()) { + throw new YarnException( + "Illegal argument! The parameter should not be null or empty"); + } + File configurationFile; + if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name)) { + URL url= classLoader.getResource(name); + if (url == null) { + throw new YarnException(name + " not found on the classpath."); + } + configurationFile = new File(url.getPath()); + } else { + configurationFile = new File(name); + } + return new ConfigurationContext(name, new FileInputStream( + configurationFile),configurationFile.lastModified()); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 6ebf90a..e55ac6d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -19,14 +19,16 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.io.IOException; +import java.lang.reflect.Method; import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.HashSet; import java.util.Map; import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.HAServiceStatus; @@ -94,6 +96,24 @@ private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); + private static final Set refreshFunctions; + static { + Set set = new HashSet(); + set.add(new RefreshContext("refreshQueues", RefreshQueuesRequest.class, + RefreshQueuesRequest.newInstance())); + set.add(new RefreshContext("refreshNodes", RefreshNodesRequest.class, + RefreshNodesRequest.newInstance())); + set.add(new RefreshContext("refreshSuperUserGroupsConfiguration", + RefreshSuperUserGroupsConfigurationRequest.class, + RefreshSuperUserGroupsConfigurationRequest.newInstance())); + set.add(new RefreshContext("refreshUserToGroupsMappings", + RefreshUserToGroupsMappingsRequest.class, + RefreshUserToGroupsMappingsRequest.newInstance())); + set.add(new RefreshContext("refreshServiceAcls", + RefreshServiceAclsRequest.class, RefreshServiceAclsRequest + .newInstance())); + refreshFunctions = Collections.unmodifiableSet(set); + } public AdminService(ResourceManager rm, RMContext rmContext) { super(AdminService.class.getName()); this.rm = rm; @@ -147,10 +167,8 @@ protected void startServer() throws Exception { if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - refreshServiceAcls( - getConfiguration(conf, - YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE), - RMPolicyProvider.getInstance()); + refreshServiceAcls(getConfiguration(conf,YarnConfiguration + .HADOOP_POLICY_CONFIGURATION_FILE),RMPolicyProvider.getInstance()); } if (rmContext.isHAEnabled()) { @@ -251,10 +269,12 @@ public synchronized void monitorHealth() @Override public synchronized void transitionToActive( HAServiceProtocol.StateChangeRequestInfo reqInfo) throws IOException { + refreshAdminAcls(); UserGroupInformation user = checkAccess("transitionToActive"); checkHaStateChange(reqInfo); try { rm.transitionToActive(); + maunallyRefreshAll(); RMAuditLogger.logSuccess(user.getShortUserName(), "transitionToActive", "RMHAProtocolService"); } catch (Exception e) { @@ -269,6 +289,7 @@ public synchronized void transitionToActive( @Override public synchronized void transitionToStandby( HAServiceProtocol.StateChangeRequestInfo reqInfo) throws IOException { + refreshAdminAcls(); UserGroupInformation user = checkAccess("transitionToStandby"); checkHaStateChange(reqInfo); try { @@ -313,9 +334,7 @@ public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request) RefreshQueuesResponse response = recordFactory.newRecordInstance(RefreshQueuesResponse.class); try { - Configuration conf = getConfiguration(getConfig(), - YarnConfiguration.CS_CONFIGURATION_FILE); - rmContext.getScheduler().reinitialize(conf, this.rmContext); + rmContext.getScheduler().reinitialize(getConfig(), this.rmContext); RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); return response; @@ -368,7 +387,7 @@ public RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfigu } Configuration conf = - getConfiguration(getConfig(), + getConfiguration(new Configuration(), YarnConfiguration.CORE_SITE_CONFIGURATION_FILE); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); RMAuditLogger.logSuccess(user.getShortUserName(), @@ -393,7 +412,7 @@ public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings( } Groups.getUserToGroupsMappingService( - getConfiguration(getConfig(), + getConfiguration(new Configuration(), YarnConfiguration.CORE_SITE_CONFIGURATION_FILE)).refresh(); RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); @@ -415,7 +434,8 @@ public RefreshAdminAclsResponse refreshAdminAcls( throwStandbyException(); } Configuration conf = - getConfiguration(getConfig(), YarnConfiguration.YARN_SITE_XML_FILE); + getConfiguration(new Configuration(), + YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); adminAcl = new AccessControlList(conf.get( YarnConfiguration.YARN_ADMIN_ACL, YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); @@ -425,6 +445,24 @@ public RefreshAdminAclsResponse refreshAdminAcls( return recordFactory.newRecordInstance(RefreshAdminAclsResponse.class); } + private void refreshAdminAcls() { + String argName = "refreshAdminAcls"; + UserGroupInformation user; + try { + user = checkAcls(argName); + Configuration conf = + getConfiguration(new Configuration(), + YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); + adminAcl = new AccessControlList(conf.get( + YarnConfiguration.YARN_ADMIN_ACL, + YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); + RMAuditLogger.logSuccess(user.getShortUserName(), argName, + "AdminService"); + } catch (Exception ex) { + LOG.warn("Fail to execute " + argName, ex); + } + } + @Override public RefreshServiceAclsResponse refreshServiceAcls( RefreshServiceAclsRequest request) throws YarnException, IOException { @@ -448,7 +486,7 @@ public RefreshServiceAclsResponse refreshServiceAcls( PolicyProvider policyProvider = RMPolicyProvider.getInstance(); Configuration conf = - getConfiguration(getConfig(), + getConfiguration(new Configuration(), YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); refreshServiceAcls(conf, policyProvider); @@ -463,13 +501,8 @@ public RefreshServiceAclsResponse refreshServiceAcls( private synchronized void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - if (this.rmContext.getConfigurationProvider() instanceof - LocalConfigurationProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); - } else { - this.server.refreshServiceAclWithLoadedConfiguration(configuration, - policyProvider); - } + this.server.refreshServiceAclWithLoadedConfiguration(configuration, + policyProvider); } @Override @@ -519,8 +552,9 @@ public UpdateNodeResourceResponse updateNodeResource( private synchronized Configuration getConfiguration(Configuration conf, String confFileName) throws YarnException, IOException { - return this.rmContext.getConfigurationProvider().getConfiguration(conf, - confFileName); + conf.addResource(this.rmContext.getConfigurationProvider() + .getConfigurationContext(confFileName).getInputStream()); + return conf; } @VisibleForTesting @@ -532,4 +566,43 @@ public AccessControlList getAccessControlList() { public Server getServer() { return this.server; } + + private void maunallyRefreshAll() { + for (RefreshContext context : refreshFunctions) { + try { + Method api = + this.getClass().getMethod(context.getFunctionName(), + context.getParameterClass()); + api.invoke(this, context.getParamterObject()); + } catch (Exception ex) { + LOG.warn("Can not execute " + context.getFunctionName(), ex); + } + } + } + + private static class RefreshContext { + + private String functionName; + private Class parameterClass; + private Object paramterObject; + + public RefreshContext(String functionName, Class parameterClass, + Object paramterObject) { + this.functionName = functionName; + this.parameterClass = parameterClass; + this.paramterObject = paramterObject; + } + + public Object getParamterObject() { + return paramterObject; + } + + public Class getParameterClass() { + return parameterClass; + } + + public String getFunctionName() { + return functionName; + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 0c56134..f593030 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -39,7 +39,6 @@ import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; @@ -140,10 +139,10 @@ protected void serviceStart() throws Exception { if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - refreshServiceAcls( - this.rmContext.getConfigurationProvider().getConfiguration(conf, - YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE), - RMPolicyProvider.getInstance()); + conf.addResource(this.rmContext.getConfigurationProvider() + .getConfigurationContext(YarnConfiguration + .HADOOP_POLICY_CONFIGURATION_FILE).getInputStream()); + refreshServiceAcls(conf, RMPolicyProvider.getInstance()); } this.server.start(); @@ -584,13 +583,8 @@ public void unregisterAttempt(ApplicationAttemptId attemptId) { public void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - if (this.rmContext.getConfigurationProvider() instanceof - LocalConfigurationProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); - } else { - this.server.refreshServiceAclWithLoadedConfiguration(configuration, - policyProvider); - } + this.server.refreshServiceAclWithLoadedConfiguration(configuration, + policyProvider); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 43e94ed..e6dd204 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -43,7 +43,6 @@ import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; @@ -171,10 +170,10 @@ protected void serviceStart() throws Exception { if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - refreshServiceAcls( - this.rmContext.getConfigurationProvider().getConfiguration(conf, - YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE), - RMPolicyProvider.getInstance()); + conf.addResource(this.rmContext.getConfigurationProvider() + .getConfigurationContext(YarnConfiguration + .HADOOP_POLICY_CONFIGURATION_FILE).getInputStream()); + refreshServiceAcls(conf, RMPolicyProvider.getInstance()); } this.server.start(); @@ -807,13 +806,8 @@ private String getRenewerForToken(Token token) void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - if (this.rmContext.getConfigurationProvider() instanceof - LocalConfigurationProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); - } else { - this.server.refreshServiceAclWithLoadedConfiguration(configuration, - policyProvider); - } + this.server.refreshServiceAclWithLoadedConfiguration(configuration, + policyProvider); } private boolean isAllowedDelegationTokenOp() throws IOException { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 3aa11c5..401ba49 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -29,8 +29,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; -import org.apache.hadoop.http.HttpConfig; -import org.apache.hadoop.http.HttpConfig.Policy; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.security.Groups; @@ -43,7 +41,6 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -190,19 +187,16 @@ protected void serviceInit(Configuration conf) throws Exception { ConfigurationProviderFactory.getConfigurationProvider(conf); this.configurationProvider.init(this.conf); rmContext.setConfigurationProvider(configurationProvider); - if (!(this.configurationProvider instanceof LocalConfigurationProvider)) { - // load yarn-site.xml - this.conf = - this.configurationProvider.getConfiguration(this.conf, - YarnConfiguration.YARN_SITE_XML_FILE); - // load core-site.xml - this.conf = - this.configurationProvider.getConfiguration(this.conf, - YarnConfiguration.CORE_SITE_CONFIGURATION_FILE); - // Do refreshUserToGroupsMappings with loaded core-site.xml - Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf) - .refresh(); - } + + // load yarn-site.xml + this.conf.addResource(this.configurationProvider.getConfigurationContext( + YarnConfiguration.YARN_SITE_CONFIGURATION_FILE).getInputStream()); + // load core-site.xml + this.conf.addResource(this.configurationProvider.getConfigurationContext( + YarnConfiguration.CORE_SITE_CONFIGURATION_FILE).getInputStream()); + // Do refreshUserToGroupsMappings with loaded core-site.xml + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf) + .refresh(); // register the handlers for all AlwaysOn services using setupDispatcher(). rmDispatcher = setupDispatcher(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 8136c05..b5728dc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -29,9 +29,7 @@ import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.VersionUtil; -import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; @@ -164,10 +162,10 @@ protected void serviceStart() throws Exception { if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - refreshServiceAcls( - this.rmContext.getConfigurationProvider().getConfiguration(conf, - YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE), - RMPolicyProvider.getInstance()); + conf.addResource(this.rmContext.getConfigurationProvider() + .getConfigurationContext(YarnConfiguration + .HADOOP_POLICY_CONFIGURATION_FILE).getInputStream()); + refreshServiceAcls(conf, RMPolicyProvider.getInstance()); } this.server.start(); @@ -421,13 +419,8 @@ public static Node resolve(String hostName) { void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - if (this.rmContext.getConfigurationProvider() instanceof - LocalConfigurationProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); - } else { - this.server.refreshServiceAclWithLoadedConfiguration(configuration, - policyProvider); - } + this.server.refreshServiceAclWithLoadedConfiguration(configuration, + policyProvider); } @VisibleForTesting diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index eb4f814..c153e40 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -35,7 +35,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -263,19 +262,8 @@ public Resource getClusterResources() { reinitialize(Configuration conf, RMContext rmContext) throws IOException { Configuration configuration = new Configuration(conf); if (!initialized) { - if (rmContext.getConfigurationProvider() instanceof - LocalConfigurationProvider) { - this.conf = new CapacitySchedulerConfiguration(configuration, true); - } else { - try { - this.conf = - new CapacitySchedulerConfiguration(rmContext - .getConfigurationProvider().getConfiguration(configuration, - YarnConfiguration.CS_CONFIGURATION_FILE), false); - } catch (Exception e) { - throw new IOException(e); - } - } + this.rmContext = rmContext; + this.conf = loadCapacitySchedulerConfiguration(configuration); validateConf(this.conf); this.minimumAllocation = this.conf.getMinimumAllocation(); this.maximumAllocation = this.conf.getMaximumAllocation(); @@ -283,7 +271,6 @@ public Resource getClusterResources() { this.usePortForNodeName = this.conf.getUsePortForNodeName(); this.applications = new ConcurrentHashMap(); - this.rmContext = rmContext; initializeQueues(this.conf); @@ -294,10 +281,7 @@ public Resource getClusterResources() { "maximumAllocation=<" + getMaximumResourceCapability() + ">"); } else { CapacitySchedulerConfiguration oldConf = this.conf; - this.conf = - new CapacitySchedulerConfiguration(conf, - rmContext.getConfigurationProvider() instanceof - LocalConfigurationProvider); + this.conf = loadCapacitySchedulerConfiguration(configuration); validateConf(this.conf); try { LOG.info("Re-initializing queues..."); @@ -1042,4 +1026,16 @@ public synchronized boolean checkAccess(UserGroupInformation callerUGI, queue.collectSchedulerApplications(apps); return apps; } + + private CapacitySchedulerConfiguration loadCapacitySchedulerConfiguration( + Configuration configuration) throws IOException { + try { + configuration.addResource(this.rmContext.getConfigurationProvider() + .getConfigurationContext(YarnConfiguration.CS_CONFIGURATION_FILE) + .getInputStream()); + return new CapacitySchedulerConfiguration(configuration, false); + } catch (Exception e) { + throw new IOException(e); + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/hadoop-policy.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/hadoop-policy.xml new file mode 100644 index 0000000..d2ddf89 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/hadoop-policy.xml @@ -0,0 +1,20 @@ + + + + + + + + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site.xml new file mode 100644 index 0000000..d2ddf89 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site.xml @@ -0,0 +1,20 @@ + + + + + + + + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/hadoop-policy.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/hadoop-policy.xml new file mode 100644 index 0000000..d2ddf89 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/hadoop-policy.xml @@ -0,0 +1,20 @@ + + + + + + + + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/yarn-site.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/yarn-site.xml new file mode 100644 index 0000000..d2ddf89 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/yarn-site.xml @@ -0,0 +1,20 @@ + + + + + + + +