diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 54daeaafbe9..6a49af05f45 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -44,6 +44,7 @@ function hadoop_usage hadoop_add_subcommand "node" admin "prints node report(s)" hadoop_add_subcommand "nodemanager" daemon "run a nodemanager on each worker" hadoop_add_subcommand "proxyserver" daemon "run the web app proxy server" + hadoop_add_subcommand "apiserver" daemon "run the yarn service api server" hadoop_add_subcommand "queue" client "prints queue information" hadoop_add_subcommand "registrydns" daemon "run the registry DNS server" hadoop_add_subcommand "resourcemanager" daemon "run the ResourceManager" @@ -139,6 +140,14 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" HADOOP_HEAPSIZE_MAX="${YARN_PROXYSERVER_HEAPSIZE}" fi ;; + apiserver) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApiServerWebApp' + # Backwards compatibility + if [[ -n "${YARN_APISERVER_HEAPSIZE}" ]]; then + HADOOP_HEAPSIZE_MAX="${YARN_APISERVER_HEAPSIZE}" + fi + ;; queue) HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.QueueCLI ;; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java index 0ccc149098c..6cd661cff86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.net.URI; import java.text.MessageFormat; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.security.PrivilegedExceptionAction; @@ -56,6 +57,7 @@ import org.apache.hadoop.yarn.service.api.records.ServiceState; import org.apache.hadoop.yarn.service.api.records.ServiceStatus; import org.apache.hadoop.yarn.service.conf.RestApiConstants; +import org.apache.hadoop.yarn.service.conf.YarnServiceConf; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.util.RMHAUtils; import org.eclipse.jetty.util.UrlEncoded; @@ -102,17 +104,24 @@ String getRMWebAddress() throws IOException { String path = "/app/v1/services/version"; String rmAddress = conf .get("yarn.resourcemanager.webapp.address"); + String apiServerAddress = conf + .get(YarnServiceConf.API_SERVER_ADDRESS, + YarnServiceConf.DEFAULT_API_SERVER_ADDRESS + + YarnServiceConf.DEFAULT_API_SERVER_PORT); if (YarnConfiguration.useHttps(conf)) { scheme = "https://"; rmAddress = conf .get("yarn.resourcemanager.webapp.https.address"); } - if (HAUtil.isHAEnabled(conf)) { boolean useKerberos = UserGroupInformation.isSecurityEnabled(); - List rmServers = getRMHAWebAddresses(conf); + List servers = new ArrayList<>(); + if(!apiServerAddress.startsWith(YarnServiceConf.DEFAULT_API_SERVER_ADDRESS)){ + servers.add(apiServerAddress); + } + servers.addAll(getRMHAWebAddresses(conf)); StringBuilder diagnosticsMsg = new StringBuilder(); - for (String host : rmServers) { + for (String host : servers) { try { Client client = Client.create(); client.setFollowRedirects(false); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java index f4acd942cc9..aa885d74c09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java @@ -22,12 +22,17 @@ import org.apache.hadoop.security.AuthenticationFilterInitializer; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.service.conf.RestApiConstants; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; -import org.eclipse.jetty.webapp.Configuration; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.eclipse.jetty.servlet.FilterMapping; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,8 +43,6 @@ import java.util.Collections; import java.util.List; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY; import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*; /** @@ -57,13 +60,16 @@ private InetSocketAddress bindAddress; public static void main(String[] args) throws IOException { + Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); + StringUtils.startupShutdownMessage(ApiServerWebApp.class, args, logger); ApiServerWebApp apiWebApp = new ApiServerWebApp(); try { apiWebApp.init(new YarnConfiguration()); apiWebApp.serviceStart(); - } catch (Exception e) { - logger.error("Got exception starting", e); + apiWebApp.apiServer.join(); + } catch (Throwable t) { apiWebApp.close(); + ExitUtil.terminate(-1, t); } } @@ -93,8 +99,8 @@ protected void serviceStop() throws Exception { private void doSecureLogin(org.apache.hadoop.conf.Configuration conf) throws IOException { - SecurityUtil.login(conf, YarnConfiguration.RM_KEYTAB, - YarnConfiguration.RM_PRINCIPAL, bindAddress.getHostName()); + SecurityUtil.login(conf, API_SERVER_KEYTAB, + API_SERVER_PRINCIPAL, bindAddress.getHostName()); addFilters(conf); } @@ -123,29 +129,46 @@ private void addFilters(org.apache.hadoop.conf.Configuration conf) { } private void startWebApp() throws IOException { - URI uri = URI.create("http://" + NetUtils.getHostPortString(bindAddress)); - apiServer = new HttpServer2.Builder() + org.apache.hadoop.conf.Configuration conf = getConfig(); + + AccessControlList acl = new AccessControlList(conf.get(YarnConfiguration.YARN_ADMIN_ACL, + YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); + + + URI uri = URI.create(WebAppUtils.getHttpSchemePrefix(conf) + NetUtils.getHostPortString(bindAddress)); + String pathSpec = "/app" + RestApiConstants.CONTEXT_ROOT + "/*"; + HttpServer2.Builder b = new HttpServer2.Builder() .setName("api-server") - .setConf(getConfig()) + .setConf(conf) .setSecurityEnabled(UserGroupInformation.isSecurityEnabled()) - .setUsernameConfKey(RM_WEBAPP_SPNEGO_USER_NAME_KEY) - .setKeytabConfKey(RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) - .addEndpoint(uri).build(); + .setUsernameConfKey(API_SERVER_SPNEGO_USER_NAME_KEY) + .setKeytabConfKey(API_SERVER_SPNEGO_KEYTAB_FILE_KEY) + .setPathSpec(new String[]{pathSpec}) + .setACL(acl) + .addEndpoint(uri); + if (YarnConfiguration.useHttps(conf)) { + WebAppUtils.loadSslConfiguration(b); + } + apiServer = b.build(); String apiPackages = ApiServer.class.getPackage().getName() + SEP + GenericExceptionHandler.class.getPackage().getName() + SEP + YarnJacksonJaxbJsonProvider.class.getPackage().getName(); - apiServer.addJerseyResourcePackage(apiPackages, "/*"); + apiServer.addJerseyResourcePackage(apiPackages, "/app/*"); try { logger.info("Service starting up. Logging start..."); apiServer.start(); logger.info("Server status = {}", apiServer.toString()); - for (Configuration conf : apiServer.getWebAppContext() + for (org.eclipse.jetty.webapp.Configuration webAppConf : apiServer.getWebAppContext() .getConfigurations()) { - logger.info("Configurations = {}", conf); + logger.info("Configurations = {}", webAppConf); + } + FilterMapping[] fmaps = apiServer.getWebAppContext().getServletHandler().getFilterMappings(); + for(FilterMapping fmap: fmaps){ + logger.info("FilterMapping = {}", fmap.toString()); } logger.info("Context Path = {}", Collections.singletonList( apiServer.getWebAppContext().getContextPath())); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java index d3716b91933..c0b4ac21fd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java @@ -59,6 +59,12 @@ public static final String DEFAULT_API_SERVER_ADDRESS = "0.0.0.0:"; public static final int DEFAULT_API_SERVER_PORT = 9191; + public static final String API_SERVER_KEYTAB = "yarn.service.api-server.keytab"; + public static final String API_SERVER_PRINCIPAL = "yarn.service.api-server.principal"; + + public static final String API_SERVER_SPNEGO_USER_NAME_KEY = "yarn.service.api-server.spnego-principal"; + public static final String API_SERVER_SPNEGO_KEYTAB_FILE_KEY = "yarn.service.api-server.spnego-keytab-file"; + public static final String FINAL_LOG_INCLUSION_PATTERN = "yarn.service.log.include-pattern"; public static final String FINAL_LOG_EXCLUSION_PATTERN = "yarn.service.log.exclude-pattern"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java index 50a1a0b6c7d..7ca9edfe770 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java @@ -214,8 +214,12 @@ static YarnConfiguration getYarnConfWithRmHaId(Configuration conf) public static String generateToken(String server) throws IOException, InterruptedException { UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); - LOG.debug("The user credential is {}", currentUser); - String challenge = currentUser + UserGroupInformation ugi = currentUser; + if(currentUser.getRealUser() != null){ + ugi = currentUser.getRealUser(); + } + LOG.debug("The user credential is {}", ugi); + String challenge = ugi .doAs(new PrivilegedExceptionAction() { @Override public String run() throws Exception {