diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java index d3349d2..2f0f2c2 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java @@ -80,6 +80,7 @@ import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; @@ -148,8 +149,14 @@ private void initializeWebApp(Configuration conf) { JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS, JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_PORT); // NOTE: there should be a .at(InetSocketAddress) - WebApps.$for("jobhistory", HistoryClientService.class, this, "ws") - .with(conf).at(NetUtils.getHostPortString(bindAddress)).start(webApp); + WebApps + .$for("jobhistory", HistoryClientService.class, this, "ws") + .with(conf) + .withHttpSpnegoKeytabKey( + YarnConfiguration.JHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) + .withHttpSpnegoPrincipalKey( + YarnConfiguration.JHS_WEBAPP_SPNEGO_USER_NAME_KEY) + .at(NetUtils.getHostPortString(bindAddress)).start(webApp); conf.updateConnectAddr(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, webApp.getListenerAddress()); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ec9eb19..f57091e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -205,6 +205,12 @@ public static final String RM_KEYTAB = RM_PREFIX + "keytab"; + public static final String RM_WEBAPP_SPNEGO_USER_NAME_KEY = + RM_PREFIX + "webapp.spnego-principal"; + + public static final String RM_WEBAPP_SPENGO_KEYTAB_FILE_KEY = + RM_PREFIX + "webapp.spengo-keytab-file"; + /** How long to wait until a container is considered dead.*/ public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = RM_PREFIX + "rm.container-allocation.expiry-interval-ms"; @@ -599,7 +605,13 @@ public static final String NM_USER_HOME_DIR = NM_PREFIX + "user-home-dir"; - + + public static final String NM_WEBAPP_SPNEGO_USER_NAME_KEY = + NM_PREFIX + "webapp.spnego-principal"; + + public static final String NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = + NM_PREFIX + "webapp.spnego-keytab-file"; + public static final String DEFAULT_NM_USER_HOME_DIR= "/home/"; //////////////////////////////// @@ -729,6 +741,12 @@ // Other Configs //////////////////////////////// + public static final String JHS_WEBAPP_SPNEGO_USER_NAME_KEY = + "jobhistoryserver.webapp.spnego-principal"; + + public static final String JHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = + "jobhistoryserver.webapp.spnego-keytab-file"; + /** * The interval of the yarn client's querying application state after * application submission. The unit is millisecond. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java index b7bf636..87622c2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java @@ -33,6 +33,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpServer; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.AdminACLsManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,7 +67,6 @@ @InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"}) public class WebApps { static final Logger LOG = LoggerFactory.getLogger(WebApps.class); - public static class Builder { static class ServletStruct { public Class clazz; @@ -82,6 +83,8 @@ boolean findPort = false; Configuration conf; boolean devMode = false; + private String spnegoPrincipalKey; + private String spnegoKeytabKey; private final HashSet servlets = new HashSet(); private final HashMap attributes = new HashMap(); @@ -135,6 +138,16 @@ this.conf = conf; return this; } + + public Builder withHttpSpnegoPrincipalKey(String spnegoPrincipalKey) { + this.spnegoPrincipalKey = spnegoPrincipalKey; + return this; + } + + public Builder withHttpSpnegoKeytabKey(String spnegoKeytabKey) { + this.spnegoKeytabKey = spnegoKeytabKey; + return this; + } public Builder inDevMode() { devMode = true; @@ -197,8 +210,30 @@ public void setup() { } } HttpServer server = - new HttpServer(name, bindAddress, port, findPort, conf, - new AdminACLsManager(conf).getAdminAcl(), null, webapp.getServePathSpecs()); + new HttpServer(name, bindAddress, port, findPort, conf, + new AdminACLsManager(conf).getAdminAcl(), null, + webapp.getServePathSpecs()) { + + { + if (UserGroupInformation.isSecurityEnabled()) { + boolean initSpnego = true; + if (spnegoPrincipalKey == null || spnegoPrincipalKey.isEmpty()) { + LOG.warn("Principal for spnego filter is not set"); + initSpnego = false; + } + if (spnegoKeytabKey == null || spnegoKeytabKey.isEmpty()) { + LOG.warn("Keytab for spnego filter is not set"); + initSpnego = false; + } + if (initSpnego) { + LOG.info("Initializing spnego filter with principal key : " + + spnegoPrincipalKey + " keytab key : " + + spnegoKeytabKey); + initSpnego(conf, spnegoPrincipalKey, spnegoKeytabKey); + } + } + } + }; for(ServletStruct struct: servlets) { server.addServlet(struct.name, struct.spec, struct.clazz); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java index 25dfaf6..fab88b9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java @@ -59,8 +59,15 @@ protected void serviceStart() throws Exception { LOG.info("Instantiating NMWebApp at " + bindAddress); try { this.webApp = - WebApps.$for("node", Context.class, this.nmContext, "ws") - .at(bindAddress).with(getConfig()).start(this.nmWebApp); + WebApps + .$for("node", Context.class, this.nmContext, "ws") + .at(bindAddress) + .with(getConfig()) + .withHttpSpnegoPrincipalKey( + YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY) + .withHttpSpnegoKeytabKey( + YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) + .start(this.nmWebApp); this.port = this.webApp.httpServer().getPort(); } catch (Exception e) { String msg = "NMWebapps failed to start."; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index a4fb30f..47d947d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -30,6 +30,7 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.service.Service; @@ -573,9 +574,16 @@ public void handle(RMNodeEvent event) { protected void startWepApp() { Builder builder = - WebApps.$for("cluster", ApplicationMasterService.class, masterService, "ws").at( - this.conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS)); + WebApps + .$for("cluster", ApplicationMasterService.class, masterService, + "ws") + .with(conf) + .withHttpSpnegoPrincipalKey( + YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY) + .withHttpSpnegoKeytabKey( + YarnConfiguration.RM_WEBAPP_SPENGO_KEYTAB_FILE_KEY) + .at(this.conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS)); String proxyHostAndPort = YarnConfiguration.getProxyHostAndPort(conf); if(YarnConfiguration.getRMWebAppHostAndPort(conf). equals(proxyHostAndPort)) {