diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java index 3751ad9..ff98cf9 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java @@ -25,6 +25,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.EnumSet; +import java.util.HashSet; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -77,10 +79,12 @@ import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; import org.apache.hadoop.mapreduce.v2.util.MRWebAppUtil; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.AuthenticationFilterInitializer; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; @@ -150,6 +154,22 @@ protected void serviceStart() throws Exception { @VisibleForTesting protected void initializeWebApp(Configuration conf) { + + // Insert the AuthenticationFilterInitializer + String filterInitializerConfKey = "hadoop.http.filter.initializers"; + String[] initializerNames = conf.getStrings(filterInitializerConfKey); + Set filterInitializers = new HashSet<>(); + if (initializerNames != null && initializerNames.length > 0) { + for (String s : initializerNames) { + filterInitializers.add(s.trim()); + } + } + + filterInitializers.add(AuthenticationFilterInitializer.class.getName()); + String actualInitializers = StringUtils.join(",", filterInitializers); + LOG.info("Using authentication filters: " + actualInitializers); + conf.set(filterInitializerConfKey, actualInitializers); + webApp = new HsWebApp(history); InetSocketAddress bindAddress = MRWebAppUtil.getJHSWebBindAddress(conf); // NOTE: there should be a .at(InetSocketAddress) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 4153ba1..819aa09 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -105,7 +105,10 @@ import java.net.InetSocketAddress; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -869,88 +872,43 @@ public void handle(RMNodeEvent event) { } } } - - protected void startWepApp() { - // Use the customized yarn filter instead of the standard kerberos filter to - // allow users to authenticate using delegation tokens - // 4 conditions need to be satisfied - - // 1. security is enabled - // 2. http auth type is set to kerberos - // 3. "yarn.resourcemanager.webapp.use-yarn-filter" override is set to true - // 4. hadoop.http.filter.initializers container AuthenticationFilterInitializer + protected void startWepApp() { Configuration conf = getConfig(); - boolean useYarnAuthenticationFilter = - conf.getBoolean( - YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER, - YarnConfiguration.DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER); - String authPrefix = "hadoop.http.authentication."; - String authTypeKey = authPrefix + "type"; + boolean useYarnAuthenticationFilter = conf.getBoolean( + YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER, + YarnConfiguration.DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER); + String filterInitializerConfKey = "hadoop.http.filter.initializers"; - String actualInitializers = ""; - Class[] initializersClasses = - conf.getClasses(filterInitializerConfKey); - - boolean hasHadoopAuthFilterInitializer = false; - boolean hasRMAuthFilterInitializer = false; - if (initializersClasses != null) { - for (Class initializer : initializersClasses) { - if (initializer.getName().equals( - AuthenticationFilterInitializer.class.getName())) { - hasHadoopAuthFilterInitializer = true; - } - if (initializer.getName().equals( - RMAuthenticationFilterInitializer.class.getName())) { - hasRMAuthFilterInitializer = true; - } + String[] initializerNames = conf.getStrings(filterInitializerConfKey); + Set filterInitializers = new HashSet<>(); + if (initializerNames != null && initializerNames.length > 0) { + for (String s : initializerNames) { + filterInitializers.add(s.trim()); } - if (UserGroupInformation.isSecurityEnabled() - && useYarnAuthenticationFilter - && hasHadoopAuthFilterInitializer - && conf.get(authTypeKey, "").equals( - KerberosAuthenticationHandler.TYPE)) { - ArrayList target = new ArrayList(); - for (Class filterInitializer : initializersClasses) { - if (filterInitializer.getName().equals( - AuthenticationFilterInitializer.class.getName())) { - if (hasRMAuthFilterInitializer == false) { - target.add(RMAuthenticationFilterInitializer.class.getName()); - } - continue; - } - target.add(filterInitializer.getName()); - } - actualInitializers = StringUtils.join(",", target); + } - LOG.info("Using RM authentication filter(kerberos/delegation-token)" - + " for RM webapp authentication"); - RMAuthenticationFilter - .setDelegationTokenSecretManager(getClientRMService().rmDTSecretManager); - conf.set(filterInitializerConfKey, actualInitializers); - } + // Always add the RMAuthenticationFilterInitializer or AuthenticationFilterInitializer + if (useYarnAuthenticationFilter) { + filterInitializers.remove((AuthenticationFilterInitializer.class.getName())); + filterInitializers.add(RMAuthenticationFilterInitializer.class.getName()); + } else { + filterInitializers.add(AuthenticationFilterInitializer.class.getName()); } + String actualInitializers = StringUtils.join(",", filterInitializers); - // if security is not enabled and the default filter initializer has not - // been set, set the initializer to include the - // RMAuthenticationFilterInitializer which in turn will set up the simple - // auth filter. - - String initializers = conf.get(filterInitializerConfKey); - if (!UserGroupInformation.isSecurityEnabled()) { - if (initializersClasses == null || initializersClasses.length == 0) { - conf.set(filterInitializerConfKey, - RMAuthenticationFilterInitializer.class.getName()); - conf.set(authTypeKey, "simple"); - } else if (initializers.equals(StaticUserWebFilter.class.getName())) { - conf.set(filterInitializerConfKey, - RMAuthenticationFilterInitializer.class.getName() + "," - + initializers); - conf.set(authTypeKey, "simple"); - } + LOG.info("Using authentication filters: " + actualInitializers); + conf.set(filterInitializerConfKey, actualInitializers); + + if (UserGroupInformation.isSecurityEnabled()) { + RMAuthenticationFilter.setDelegationTokenSecretManager( + getClientRMService().rmDTSecretManager); + } else { + conf.set("hadoop.http.authentication.type", "simple"); } - Builder builder = + Builder builder = WebApps .$for("cluster", ApplicationMasterService.class, masterService, "ws") @@ -973,7 +931,6 @@ protected void startWepApp() { builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher); String[] proxyParts = proxyHostAndPort.split(":"); builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, proxyParts[0]); - } webApp = builder.start(new RMWebApp(this)); }