diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationContext.java
new file mode 100644
index 0000000..c3ff903
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationContext.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.conf;
+
+import java.io.InputStream;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ *
ConfigurationContext represents all needed
+ * information of a given Configuration file.
+ *
+ * It includes details such as:
+ *
+ * - Configuration name
+ * - InputStream for all the contents
+ * - last Modified Time
+ *
+ *
+ * @see ConfigurationProvider#getConfigurationContext(String)
+ */
+@Private
+@Unstable
+public class ConfigurationContext {
+ private String name;
+ private InputStream inputStream;
+ private long lastModifiedTime;
+
+ public ConfigurationContext(String name, InputStream inputStream,
+ long lastModifiedTime) {
+ this.name = name;
+ this.inputStream = inputStream;
+ this.lastModifiedTime = lastModifiedTime;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public InputStream getInputStream() {
+ return inputStream;
+ }
+
+ public void setInputStream(InputStream inputStream) {
+ this.inputStream = inputStream;
+ }
+
+ public long getLastModifiedTime() {
+ return lastModifiedTime;
+ }
+
+ public void setLastModifiedTime(long lastModifiedTime) {
+ this.lastModifiedTime = lastModifiedTime;
+ }
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java
index b31573d..c173a19 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java
@@ -42,15 +42,14 @@ public void close() throws Exception {
}
/**
- * Get the configuration and combine with bootstrapConf
- * @param bootstrapConf Configuration
+ * create a {@link ConfigurationContext} object for the indicated Configuration file
* @param name The configuration file name
- * @return configuration
+ * @return configurationContext
* @throws YarnException
* @throws IOException
*/
- public abstract Configuration getConfiguration(Configuration bootstrapConf,
- String name) throws YarnException, IOException;
+ public abstract ConfigurationContext getConfigurationContext(String name)
+ throws YarnException, IOException;
/**
* Derived classes initialize themselves using this method.
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 9612cac..f07a3e3 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -45,22 +45,30 @@
"hadoop-policy.xml";
@Private
- public static final String YARN_SITE_XML_FILE = "yarn-site.xml";
+ public static final String YARN_SITE_CONFIGURATION_FILE = "yarn-site.xml";
+ private static final String YARN_DEFAULT_CONFIGURATION_FILE =
+ "yarn-default.xml";
@Private
public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml";
+ @Private
+ public static final List RM_CONFIGURATION_FILES =
+ Collections.unmodifiableList(Arrays.asList(
+ CS_CONFIGURATION_FILE,
+ HADOOP_POLICY_CONFIGURATION_FILE,
+ YARN_SITE_CONFIGURATION_FILE,
+ CORE_SITE_CONFIGURATION_FILE));
+
@Evolving
public static final int APPLICATION_MAX_TAGS = 10;
@Evolving
public static final int APPLICATION_MAX_TAG_LENGTH = 100;
- private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml";
-
static {
- Configuration.addDefaultResource(YARN_DEFAULT_XML_FILE);
- Configuration.addDefaultResource(YARN_SITE_XML_FILE);
+ Configuration.addDefaultResource(YARN_DEFAULT_CONFIGURATION_FILE);
+ Configuration.addDefaultResource(YARN_SITE_CONFIGURATION_FILE);
}
//Configurations
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
index a57d507..1713798 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
@@ -23,6 +23,9 @@
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
@@ -30,6 +33,8 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.ha.ClientBaseWithFixes;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.proto.HAServiceProtocolProtos;
@@ -41,7 +46,10 @@
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest;
import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer;
import org.junit.After;
import org.junit.Assert;
@@ -63,6 +71,8 @@
private Configuration conf;
private MiniYARNCluster cluster;
private ApplicationId fakeAppId;
+ private FileSystem fs;
+ private Path workingPath;
private void setConfForRM(String rmId, String prefix, String value) {
@@ -98,6 +108,15 @@ public void setup() throws IOException {
conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);
+ fs = FileSystem.get(conf);
+ workingPath =
+ new Path(new File("target", this.getClass().getSimpleName()
+ + "-remoteDir").getAbsolutePath());
+ conf.set(YarnConfiguration.FS_BASED_RM_CONF_STORE,
+ workingPath.toString());
+ fs.delete(workingPath, true);
+ fs.mkdirs(workingPath);
+
cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 1, 1, 1);
}
@@ -245,6 +264,47 @@ public void testEmbeddedWebAppProxy() throws YarnException,
verifyExpectedException(proxyConn.getResponseMessage());
}
+ @Test
+ public void testRMFailOverWithFileSystemBasedConfigurationProvider()
+ throws IOException, YarnException, InterruptedException {
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
+ conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
+ // upload default configurations
+ uploadDefaultConfiguration();
+ YarnConfiguration yarnConf = new YarnConfiguration(conf);
+ uploadConfiguration(yarnConf, "yarn-site.xml");
+
+ cluster.init(conf);
+ cluster.start();
+ getAdminService(0).transitionToActive(req);
+ assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex());
+
+ CapacityScheduler cs =
+ (CapacityScheduler) cluster.getResourceManager(0).getRMContext()
+ .getScheduler();
+ int maxAppsBefore = cs.getConfiguration().getMaximumSystemApplications();
+
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+ csConf.set("yarn.scheduler.capacity.maximum-applications", "5000");
+ uploadConfiguration(csConf, "capacity-scheduler.xml");
+
+ cluster.getResourceManager(0).getRMContext().getRMAdminService()
+ .refreshQueues(RefreshQueuesRequest.newInstance());
+
+ int maxAppsAfter = cs.getConfiguration().getMaximumSystemApplications();
+ Assert.assertEquals(maxAppsAfter, 5000);
+ Assert.assertTrue(maxAppsAfter != maxAppsBefore);
+
+ explicitFailover();
+ verifyConnections();
+ int maxAppsAfterFailOver =
+ ((CapacityScheduler) cluster.getResourceManager(1).getRMContext()
+ .getScheduler()).getConfiguration().getMaximumSystemApplications();
+ Assert.assertEquals(maxAppsAfterFailOver, 5000);
+ }
+
private void verifyExpectedException(String exceptionMessage){
assertTrue(exceptionMessage.contains(ApplicationNotFoundException.class
.getName()));
@@ -252,4 +312,57 @@ private void verifyExpectedException(String exceptionMessage){
.contains("Application with id '" + fakeAppId + "' " +
"doesn't exist in RM."));
}
+
+ private String writeConfigurationXML(Configuration conf, String confXMLName)
+ throws IOException {
+ DataOutputStream output = null;
+ try {
+ final File confFile = new File(tmpDir.toString(), confXMLName);
+ if (confFile.exists()) {
+ confFile.delete();
+ }
+ if (!confFile.createNewFile()) {
+ Assert.fail("Can not create " + confXMLName);
+ }
+ output = new DataOutputStream(
+ new FileOutputStream(confFile));
+ conf.writeXml(output);
+ return confFile.getAbsolutePath();
+ } finally {
+ if (output != null) {
+ output.close();
+ }
+ }
+ }
+
+ private void uploadToRemoteFileSystem(Path filePath)
+ throws IOException {
+ fs.copyFromLocalFile(filePath, workingPath);
+ }
+
+ private void uploadConfiguration(Configuration conf, String confFileName)
+ throws IOException {
+ String csConfFile = writeConfigurationXML(conf, confFileName);
+ // upload the file into Remote File System
+ uploadToRemoteFileSystem(new Path(csConfFile));
+ }
+
+ private void uploadDefaultConfiguration() throws IOException {
+ Configuration conf = new Configuration();
+ uploadConfiguration(conf, "core-site.xml");
+
+ YarnConfiguration yarnConf = new YarnConfiguration();
+ yarnConf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
+ uploadConfiguration(yarnConf, "yarn-site.xml");
+
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+ uploadConfiguration(csConf, "capacity-scheduler.xml");
+
+ Configuration hadoopPolicyConf = new Configuration(false);
+ hadoopPolicyConf
+ .addResource(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
+ uploadConfiguration(hadoopPolicyConf, "hadoop-policy.xml");
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
index 390aace..8bbbbbc 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.conf.ConfigurationContext;
import org.apache.hadoop.yarn.conf.ConfigurationProvider;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -41,15 +42,27 @@
private Path configDir;
@Override
- public synchronized Configuration getConfiguration(Configuration bootstrapConf,
- String name) throws IOException, YarnException {
- Path configPath = new Path(this.configDir, name);
- if (!fs.exists(configPath)) {
- throw new YarnException("Can not find Configuration: " + name + " in "
- + configDir);
+ public synchronized ConfigurationContext getConfigurationContext(String name)
+ throws IOException, YarnException {
+ if (name == null || name.isEmpty()) {
+ throw new YarnException(
+ "Illegal argument! The parameter should not be null or empty");
}
- bootstrapConf.addResource(fs.open(configPath));
- return bootstrapConf;
+ Path filePath;
+ if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name)) {
+ filePath = new Path(this.configDir, name);
+ if (!fs.exists(filePath)) {
+ throw new YarnException("Can not find Configuration: " + name + " in "
+ + configDir);
+ }
+ } else {
+ filePath = new Path(name);
+ if (!fs.exists(filePath)) {
+ throw new YarnException("Can not find file: " + name);
+ }
+ }
+ return new ConfigurationContext(name, fs.open(filePath), fs.getFileStatus(
+ filePath).getModificationTime());
}
@Override
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java
index 3e69960..2e4ce66 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java
@@ -18,21 +18,49 @@
package org.apache.hadoop.yarn;
+import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
+import java.net.URL;
+
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.ConfigurationContext;
import org.apache.hadoop.yarn.conf.ConfigurationProvider;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@Private
@Unstable
public class LocalConfigurationProvider extends ConfigurationProvider {
+ private ClassLoader classLoader;
+ {
+ classLoader = Thread.currentThread().getContextClassLoader();
+ if (classLoader == null) {
+ classLoader = Configuration.class.getClassLoader();
+ }
+ }
@Override
- public Configuration getConfiguration(Configuration bootstrapConf,
- String name) throws IOException, YarnException {
- return bootstrapConf;
+ public ConfigurationContext getConfigurationContext(String name)
+ throws IOException, YarnException {
+ if (name == null || name.isEmpty()) {
+ throw new YarnException(
+ "Illegal argument! The parameter should not be null or empty");
+ }
+ File configurationFile;
+ if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name)) {
+ URL url= classLoader.getResource(name);
+ if (url == null) {
+ throw new YarnException(name + " not found on the classpath.");
+ }
+ configurationFile = new File(url.getPath());
+ } else {
+ configurationFile = new File(name);
+ }
+ return new ConfigurationContext(name, new FileInputStream(
+ configurationFile),configurationFile.lastModified());
}
@Override
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 6ebf90a..e55ac6d 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -19,14 +19,16 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
+import java.lang.reflect.Method;
import java.net.InetSocketAddress;
+import java.util.Collections;
+import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceStatus;
@@ -94,6 +96,24 @@
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
+ private static final Set refreshFunctions;
+ static {
+ Set set = new HashSet();
+ set.add(new RefreshContext("refreshQueues", RefreshQueuesRequest.class,
+ RefreshQueuesRequest.newInstance()));
+ set.add(new RefreshContext("refreshNodes", RefreshNodesRequest.class,
+ RefreshNodesRequest.newInstance()));
+ set.add(new RefreshContext("refreshSuperUserGroupsConfiguration",
+ RefreshSuperUserGroupsConfigurationRequest.class,
+ RefreshSuperUserGroupsConfigurationRequest.newInstance()));
+ set.add(new RefreshContext("refreshUserToGroupsMappings",
+ RefreshUserToGroupsMappingsRequest.class,
+ RefreshUserToGroupsMappingsRequest.newInstance()));
+ set.add(new RefreshContext("refreshServiceAcls",
+ RefreshServiceAclsRequest.class, RefreshServiceAclsRequest
+ .newInstance()));
+ refreshFunctions = Collections.unmodifiableSet(set);
+ }
public AdminService(ResourceManager rm, RMContext rmContext) {
super(AdminService.class.getName());
this.rm = rm;
@@ -147,10 +167,8 @@ protected void startServer() throws Exception {
if (conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
- refreshServiceAcls(
- getConfiguration(conf,
- YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE),
- RMPolicyProvider.getInstance());
+ refreshServiceAcls(getConfiguration(conf,YarnConfiguration
+ .HADOOP_POLICY_CONFIGURATION_FILE),RMPolicyProvider.getInstance());
}
if (rmContext.isHAEnabled()) {
@@ -251,10 +269,12 @@ public synchronized void monitorHealth()
@Override
public synchronized void transitionToActive(
HAServiceProtocol.StateChangeRequestInfo reqInfo) throws IOException {
+ refreshAdminAcls();
UserGroupInformation user = checkAccess("transitionToActive");
checkHaStateChange(reqInfo);
try {
rm.transitionToActive();
+ maunallyRefreshAll();
RMAuditLogger.logSuccess(user.getShortUserName(),
"transitionToActive", "RMHAProtocolService");
} catch (Exception e) {
@@ -269,6 +289,7 @@ public synchronized void transitionToActive(
@Override
public synchronized void transitionToStandby(
HAServiceProtocol.StateChangeRequestInfo reqInfo) throws IOException {
+ refreshAdminAcls();
UserGroupInformation user = checkAccess("transitionToStandby");
checkHaStateChange(reqInfo);
try {
@@ -313,9 +334,7 @@ public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request)
RefreshQueuesResponse response =
recordFactory.newRecordInstance(RefreshQueuesResponse.class);
try {
- Configuration conf = getConfiguration(getConfig(),
- YarnConfiguration.CS_CONFIGURATION_FILE);
- rmContext.getScheduler().reinitialize(conf, this.rmContext);
+ rmContext.getScheduler().reinitialize(getConfig(), this.rmContext);
RMAuditLogger.logSuccess(user.getShortUserName(), argName,
"AdminService");
return response;
@@ -368,7 +387,7 @@ public RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfigu
}
Configuration conf =
- getConfiguration(getConfig(),
+ getConfiguration(new Configuration(),
YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
RMAuditLogger.logSuccess(user.getShortUserName(),
@@ -393,7 +412,7 @@ public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings(
}
Groups.getUserToGroupsMappingService(
- getConfiguration(getConfig(),
+ getConfiguration(new Configuration(),
YarnConfiguration.CORE_SITE_CONFIGURATION_FILE)).refresh();
RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService");
@@ -415,7 +434,8 @@ public RefreshAdminAclsResponse refreshAdminAcls(
throwStandbyException();
}
Configuration conf =
- getConfiguration(getConfig(), YarnConfiguration.YARN_SITE_XML_FILE);
+ getConfiguration(new Configuration(),
+ YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
adminAcl = new AccessControlList(conf.get(
YarnConfiguration.YARN_ADMIN_ACL,
YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
@@ -425,6 +445,24 @@ public RefreshAdminAclsResponse refreshAdminAcls(
return recordFactory.newRecordInstance(RefreshAdminAclsResponse.class);
}
+ private void refreshAdminAcls() {
+ String argName = "refreshAdminAcls";
+ UserGroupInformation user;
+ try {
+ user = checkAcls(argName);
+ Configuration conf =
+ getConfiguration(new Configuration(),
+ YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
+ adminAcl = new AccessControlList(conf.get(
+ YarnConfiguration.YARN_ADMIN_ACL,
+ YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
+ RMAuditLogger.logSuccess(user.getShortUserName(), argName,
+ "AdminService");
+ } catch (Exception ex) {
+ LOG.warn("Fail to execute " + argName, ex);
+ }
+ }
+
@Override
public RefreshServiceAclsResponse refreshServiceAcls(
RefreshServiceAclsRequest request) throws YarnException, IOException {
@@ -448,7 +486,7 @@ public RefreshServiceAclsResponse refreshServiceAcls(
PolicyProvider policyProvider = RMPolicyProvider.getInstance();
Configuration conf =
- getConfiguration(getConfig(),
+ getConfiguration(new Configuration(),
YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
refreshServiceAcls(conf, policyProvider);
@@ -463,13 +501,8 @@ public RefreshServiceAclsResponse refreshServiceAcls(
private synchronized void refreshServiceAcls(Configuration configuration,
PolicyProvider policyProvider) {
- if (this.rmContext.getConfigurationProvider() instanceof
- LocalConfigurationProvider) {
- this.server.refreshServiceAcl(configuration, policyProvider);
- } else {
- this.server.refreshServiceAclWithLoadedConfiguration(configuration,
- policyProvider);
- }
+ this.server.refreshServiceAclWithLoadedConfiguration(configuration,
+ policyProvider);
}
@Override
@@ -519,8 +552,9 @@ public UpdateNodeResourceResponse updateNodeResource(
private synchronized Configuration getConfiguration(Configuration conf,
String confFileName) throws YarnException, IOException {
- return this.rmContext.getConfigurationProvider().getConfiguration(conf,
- confFileName);
+ conf.addResource(this.rmContext.getConfigurationProvider()
+ .getConfigurationContext(confFileName).getInputStream());
+ return conf;
}
@VisibleForTesting
@@ -532,4 +566,43 @@ public AccessControlList getAccessControlList() {
public Server getServer() {
return this.server;
}
+
+ private void maunallyRefreshAll() {
+ for (RefreshContext context : refreshFunctions) {
+ try {
+ Method api =
+ this.getClass().getMethod(context.getFunctionName(),
+ context.getParameterClass());
+ api.invoke(this, context.getParamterObject());
+ } catch (Exception ex) {
+ LOG.warn("Can not execute " + context.getFunctionName(), ex);
+ }
+ }
+ }
+
+ private static class RefreshContext {
+
+ private String functionName;
+ private Class> parameterClass;
+ private Object paramterObject;
+
+ public RefreshContext(String functionName, Class> parameterClass,
+ Object paramterObject) {
+ this.functionName = functionName;
+ this.parameterClass = parameterClass;
+ this.paramterObject = paramterObject;
+ }
+
+ public Object getParamterObject() {
+ return paramterObject;
+ }
+
+ public Class> getParameterClass() {
+ return parameterClass;
+ }
+
+ public String getFunctionName() {
+ return functionName;
+ }
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 0c56134..f593030 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -39,7 +39,6 @@
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
@@ -140,10 +139,10 @@ protected void serviceStart() throws Exception {
if (conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
- refreshServiceAcls(
- this.rmContext.getConfigurationProvider().getConfiguration(conf,
- YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE),
- RMPolicyProvider.getInstance());
+ conf.addResource(this.rmContext.getConfigurationProvider()
+ .getConfigurationContext(YarnConfiguration
+ .HADOOP_POLICY_CONFIGURATION_FILE).getInputStream());
+ refreshServiceAcls(conf, RMPolicyProvider.getInstance());
}
this.server.start();
@@ -584,13 +583,8 @@ public void unregisterAttempt(ApplicationAttemptId attemptId) {
public void refreshServiceAcls(Configuration configuration,
PolicyProvider policyProvider) {
- if (this.rmContext.getConfigurationProvider() instanceof
- LocalConfigurationProvider) {
- this.server.refreshServiceAcl(configuration, policyProvider);
- } else {
- this.server.refreshServiceAclWithLoadedConfiguration(configuration,
- policyProvider);
- }
+ this.server.refreshServiceAclWithLoadedConfiguration(configuration,
+ policyProvider);
}
@Override
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 43e94ed..e6dd204 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -43,7 +43,6 @@
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
@@ -171,10 +170,10 @@ protected void serviceStart() throws Exception {
if (conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
- refreshServiceAcls(
- this.rmContext.getConfigurationProvider().getConfiguration(conf,
- YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE),
- RMPolicyProvider.getInstance());
+ conf.addResource(this.rmContext.getConfigurationProvider()
+ .getConfigurationContext(YarnConfiguration
+ .HADOOP_POLICY_CONFIGURATION_FILE).getInputStream());
+ refreshServiceAcls(conf, RMPolicyProvider.getInstance());
}
this.server.start();
@@ -807,13 +806,8 @@ private String getRenewerForToken(Token token)
void refreshServiceAcls(Configuration configuration,
PolicyProvider policyProvider) {
- if (this.rmContext.getConfigurationProvider() instanceof
- LocalConfigurationProvider) {
- this.server.refreshServiceAcl(configuration, policyProvider);
- } else {
- this.server.refreshServiceAclWithLoadedConfiguration(configuration,
- policyProvider);
- }
+ this.server.refreshServiceAclWithLoadedConfiguration(configuration,
+ policyProvider);
}
private boolean isAllowedDelegationTokenOp() throws IOException {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 3aa11c5..401ba49 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -29,8 +29,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
-import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.http.HttpConfig.Policy;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.security.Groups;
@@ -43,7 +41,6 @@
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -190,19 +187,16 @@ protected void serviceInit(Configuration conf) throws Exception {
ConfigurationProviderFactory.getConfigurationProvider(conf);
this.configurationProvider.init(this.conf);
rmContext.setConfigurationProvider(configurationProvider);
- if (!(this.configurationProvider instanceof LocalConfigurationProvider)) {
- // load yarn-site.xml
- this.conf =
- this.configurationProvider.getConfiguration(this.conf,
- YarnConfiguration.YARN_SITE_XML_FILE);
- // load core-site.xml
- this.conf =
- this.configurationProvider.getConfiguration(this.conf,
- YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
- // Do refreshUserToGroupsMappings with loaded core-site.xml
- Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf)
- .refresh();
- }
+
+ // load yarn-site.xml
+ this.conf.addResource(this.configurationProvider.getConfigurationContext(
+ YarnConfiguration.YARN_SITE_CONFIGURATION_FILE).getInputStream());
+ // load core-site.xml
+ this.conf.addResource(this.configurationProvider.getConfigurationContext(
+ YarnConfiguration.CORE_SITE_CONFIGURATION_FILE).getInputStream());
+ // Do refreshUserToGroupsMappings with loaded core-site.xml
+ Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf)
+ .refresh();
// register the handlers for all AlwaysOn services using setupDispatcher().
rmDispatcher = setupDispatcher();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index 8136c05..b5728dc 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -29,9 +29,7 @@
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.VersionUtil;
-import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -164,10 +162,10 @@ protected void serviceStart() throws Exception {
if (conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
- refreshServiceAcls(
- this.rmContext.getConfigurationProvider().getConfiguration(conf,
- YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE),
- RMPolicyProvider.getInstance());
+ conf.addResource(this.rmContext.getConfigurationProvider()
+ .getConfigurationContext(YarnConfiguration
+ .HADOOP_POLICY_CONFIGURATION_FILE).getInputStream());
+ refreshServiceAcls(conf, RMPolicyProvider.getInstance());
}
this.server.start();
@@ -421,13 +419,8 @@ public static Node resolve(String hostName) {
void refreshServiceAcls(Configuration configuration,
PolicyProvider policyProvider) {
- if (this.rmContext.getConfigurationProvider() instanceof
- LocalConfigurationProvider) {
- this.server.refreshServiceAcl(configuration, policyProvider);
- } else {
- this.server.refreshServiceAclWithLoadedConfiguration(configuration,
- policyProvider);
- }
+ this.server.refreshServiceAclWithLoadedConfiguration(configuration,
+ policyProvider);
}
@VisibleForTesting
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index eb4f814..c153e40 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -35,7 +35,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
@@ -263,19 +262,8 @@ public Resource getClusterResources() {
reinitialize(Configuration conf, RMContext rmContext) throws IOException {
Configuration configuration = new Configuration(conf);
if (!initialized) {
- if (rmContext.getConfigurationProvider() instanceof
- LocalConfigurationProvider) {
- this.conf = new CapacitySchedulerConfiguration(configuration, true);
- } else {
- try {
- this.conf =
- new CapacitySchedulerConfiguration(rmContext
- .getConfigurationProvider().getConfiguration(configuration,
- YarnConfiguration.CS_CONFIGURATION_FILE), false);
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
+ this.rmContext = rmContext;
+ this.conf = loadCapacitySchedulerConfiguration(configuration);
validateConf(this.conf);
this.minimumAllocation = this.conf.getMinimumAllocation();
this.maximumAllocation = this.conf.getMaximumAllocation();
@@ -283,7 +271,6 @@ public Resource getClusterResources() {
this.usePortForNodeName = this.conf.getUsePortForNodeName();
this.applications =
new ConcurrentHashMap();
- this.rmContext = rmContext;
initializeQueues(this.conf);
@@ -294,10 +281,7 @@ public Resource getClusterResources() {
"maximumAllocation=<" + getMaximumResourceCapability() + ">");
} else {
CapacitySchedulerConfiguration oldConf = this.conf;
- this.conf =
- new CapacitySchedulerConfiguration(conf,
- rmContext.getConfigurationProvider() instanceof
- LocalConfigurationProvider);
+ this.conf = loadCapacitySchedulerConfiguration(configuration);
validateConf(this.conf);
try {
LOG.info("Re-initializing queues...");
@@ -1042,4 +1026,16 @@ public synchronized boolean checkAccess(UserGroupInformation callerUGI,
queue.collectSchedulerApplications(apps);
return apps;
}
+
+ private CapacitySchedulerConfiguration loadCapacitySchedulerConfiguration(
+ Configuration configuration) throws IOException {
+ try {
+ configuration.addResource(this.rmContext.getConfigurationProvider()
+ .getConfigurationContext(YarnConfiguration.CS_CONFIGURATION_FILE)
+ .getInputStream());
+ return new CapacitySchedulerConfiguration(configuration, false);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/hadoop-policy.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/hadoop-policy.xml
new file mode 100644
index 0000000..d2ddf89
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/hadoop-policy.xml
@@ -0,0 +1,20 @@
+
+
+
+
+
+
+
+
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site.xml
new file mode 100644
index 0000000..d2ddf89
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site.xml
@@ -0,0 +1,20 @@
+
+
+
+
+
+
+
+
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/hadoop-policy.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/hadoop-policy.xml
new file mode 100644
index 0000000..d2ddf89
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/hadoop-policy.xml
@@ -0,0 +1,20 @@
+
+
+
+
+
+
+
+
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/yarn-site.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/yarn-site.xml
new file mode 100644
index 0000000..d2ddf89
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/yarn-site.xml
@@ -0,0 +1,20 @@
+
+
+
+
+
+
+
+