diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml index 36364bf2a80..2899c5358bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml @@ -212,6 +212,12 @@ hadoop-minikdc test + + org.apache.hadoop + hadoop-auth + test-jar + test + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java index 834bb03fac7..23276df7904 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java @@ -20,30 +20,33 @@ import java.io.File; import java.io.IOException; -import java.net.URI; +import java.net.HttpURLConnection; +import java.net.URL; import java.text.MessageFormat; import java.util.List; import java.util.Map; -import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.UriBuilder; import com.google.common.base.Preconditions; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; +import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; import org.apache.commons.codec.binary.Base64; import com.google.common.base.Strings; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.client.api.AppAdminClient; import org.apache.hadoop.yarn.client.api.YarnClient; -import org.apache.hadoop.yarn.client.util.YarnClientUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.service.api.records.Component; @@ -56,7 +59,6 @@ import org.apache.hadoop.yarn.service.conf.RestApiConstants; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.util.RMHAUtils; -import org.eclipse.jetty.util.UrlEncoded; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,6 +78,8 @@ LoggerFactory.getLogger(ApiServiceClient.class); private static final Base64 BASE_64_CODEC = new Base64(0); protected YarnClient yarnClient; + private Client client; + private SSLFactory clientSslFactory; public ApiServiceClient() { } @@ -88,9 +92,40 @@ public ApiServiceClient(Configuration c) throws Exception { throws Exception { yarnClient = YarnClient.createYarnClient(); addService(yarnClient); + client = createClient(); + if(YarnConfiguration.useHttps(configuration)) { + clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, configuration); + } super.serviceInit(configuration); } + private Client createClient() { + Client jerseyClient = new Client( + new URLConnectionClientHandler(new HttpURLConnectionFactory() { + @Override + public HttpURLConnection getHttpURLConnection(URL url) + throws IOException { + AuthenticatedURL.Token token = new AuthenticatedURL.Token(); + AuthenticatedURL aUrl; + HttpURLConnection connection; + try { + if (clientSslFactory != null) { + clientSslFactory.init(); + aUrl = new AuthenticatedURL(null, clientSslFactory); + } else { + aUrl = new AuthenticatedURL(); + } + connection = aUrl.openConnection(url, token); + } catch (Exception e) { + throw new IOException(e); + } + return connection; + } + }), getClientConfig()); + jerseyClient.setChunkedEncodingSize(null); + return jerseyClient; + } + /** * Calculate Resource Manager address base on working REST API. */ @@ -105,34 +140,16 @@ String getRMWebAddress() { rmAddress = conf .get("yarn.resourcemanager.webapp.https.address"); } - boolean useKerberos = UserGroupInformation.isSecurityEnabled(); List rmServers = getRMHAWebAddresses(conf); for (String host : rmServers) { try { - Client client = Client.create(); - client.setFollowRedirects(false); StringBuilder sb = new StringBuilder(); sb.append(scheme) .append(host) .append(path); - if (!useKerberos) { - try { - String username = UserGroupInformation.getCurrentUser().getShortUserName(); - sb.append("?user.name=") - .append(username); - } catch (IOException e) { - LOG.debug("Fail to resolve username: {}", e); - } - } - Builder builder = client - .resource(sb.toString()).type(MediaType.APPLICATION_JSON); - if (useKerberos) { - String[] server = host.split(":"); - String challenge = YarnClientUtils.generateToken(server[0]); - builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " + - challenge); - LOG.debug("Authorization: Negotiate {}", challenge); - } + WebResource rsrc = client.resource(sb.toString()); + rsrc.setProperty(ClientConfig.PROPERTY_FOLLOW_REDIRECTS, false); + Builder builder = rsrc.type(MediaType.APPLICATION_JSON); ClientResponse test = builder.get(ClientResponse.class); if (test.getStatus() == 200) { rmAddress = host; @@ -167,7 +184,6 @@ public String getServicePath(String appName) throws IOException { api.append("/") .append(appName); } - appendUserNameIfRequired(api); return api.toString(); } @@ -178,7 +194,6 @@ private String getInstancesPath(String appName) throws IOException { api.append(url) .append("/app/v1/services/").append(appName).append("/") .append(RestApiConstants.COMP_INSTANCES); - appendUserNameIfRequired(api); return api.toString(); } @@ -206,22 +221,9 @@ private String getComponentsPath(String appName) throws IOException { api.append(url) .append("/app/v1/services/").append(appName).append("/") .append(RestApiConstants.COMPONENTS); - appendUserNameIfRequired(api); return api.toString(); } - private void appendUserNameIfRequired(StringBuilder builder) - throws IOException { - Configuration conf = getConfig(); - if (conf.get("hadoop.http.authentication.type") - .equalsIgnoreCase("simple")) { - String username = UserGroupInformation.getCurrentUser() - .getShortUserName(); - builder.append("?user.name=").append(UrlEncoded - .encodeString(username)); - } - } - public Builder getApiClient() throws IOException { return getApiClient(getServicePath(null)); } @@ -233,23 +235,10 @@ public Builder getApiClient() throws IOException { * @return * @throws IOException */ - public Builder getApiClient(String requestPath) - throws IOException { - Client client = Client.create(getClientConfig()); - client.setChunkedEncodingSize(null); - Builder builder = client - .resource(requestPath).type(MediaType.APPLICATION_JSON); - if (UserGroupInformation.isSecurityEnabled()) { - try { - URI url = new URI(requestPath); - String challenge = YarnClientUtils.generateToken(url.getHost()); - builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " + challenge); - } catch (Exception e) { - throw new IOException(e); - } - } - return builder - .accept("application/json;charset=utf-8"); + public Builder getApiClient(String requestPath) throws IOException { + Builder builder = client.resource(requestPath) + .type(MediaType.APPLICATION_JSON); + return builder.accept("application/json;charset=utf-8"); } private ClientConfig getClientConfig() { @@ -727,4 +716,14 @@ public int actionDecommissionInstances(String appName, List } return result; } + + @Override + protected void serviceStop() throws Exception { + if(client != null) { + client.destroy(); + } + if(clientSslFactory != null) { + clientSslFactory.destroy(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java index 0ffeb456c85..6292a7aba18 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java @@ -311,13 +311,4 @@ public void testComponentsUpgrade() { } } - @Test - public void testNoneSecureApiClient() throws IOException { - String url = asc.getServicePath("/foobar"); - assertTrue("User.name flag is missing in service path.", - url.contains("user.name")); - assertTrue("User.name flag is not matching JVM user.", - url.contains(System.getProperty("user.name"))); - } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java index 1ec8d41bbaf..fcbff0b8e74 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java @@ -18,46 +18,52 @@ package org.apache.hadoop.yarn.service.client; -import static org.junit.Assert.*; - -import java.io.File; -import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.minikdc.KerberosSecurityTestcase; +import org.apache.hadoop.security.AuthenticationFilterInitializer; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.KerberosTestUtils; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import javax.security.sasl.Sasl; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; - -import java.util.Map; -import java.util.ArrayList; -import java.util.Enumeration; +import java.io.File; +import java.io.IOException; +import java.net.URI; import java.util.HashMap; -import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.minikdc.KerberosSecurityTestcase; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.SaslRpcServer.QualityOfProtection; -import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; -import org.apache.hadoop.yarn.client.util.YarnClientUtils; -import org.apache.log4j.Logger; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; -import org.eclipse.jetty.util.thread.QueuedThreadPool; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import static org.apache.hadoop.yarn.service.exceptions. + LauncherExitCodes.EXIT_SUCCESS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; /** - * Test Spnego Client Login. + * Test Spnego and SSL Client Login. */ public class TestSecureApiServiceClient extends KerberosSecurityTestcase { - private String clientPrincipal = "client"; + private static final Logger LOG = + LoggerFactory.getLogger(TestSecureApiServiceClient.class); + + private static final File BASEDIR = GenericTestUtils.getRandomizedTestDir(); + + private static String clientPrincipal = "client"; private String server1Protocol = "HTTP"; @@ -69,19 +75,22 @@ private String server2Principal = server2Protocol + "/" + host; - private File keytabFile; + private String keytabConfKey = "apiservice.keytab"; + + private String principalConfKey = "apiservice.principal"; private Configuration testConf = new Configuration(); - private Map props; - private static Server server; - private static Logger LOG = Logger - .getLogger(TestSecureApiServiceClient.class); + private static String keystoresDir; + private static String sslConfDir; + private static Configuration sslConf; + private static final String PREFIX = "hadoop.http.authentication."; + + private static HttpServer2 server; private ApiServiceClient asc; /** * A mocked version of API Service for testing purpose. - * */ @SuppressWarnings("serial") public static class TestServlet extends HttpServlet { @@ -91,98 +100,222 @@ @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - Enumeration headers = req.getHeaderNames(); - while(headers.hasMoreElements()) { - String header = headers.nextElement(); - LOG.info(header); - } - if (req.getHeader("Authorization")!=null) { - headerFound = true; - resp.setStatus(HttpServletResponse.SC_OK); - } else { - headerFound = false; - resp.setStatus(HttpServletResponse.SC_NOT_FOUND); - } + checkLoginAndRespond(req, resp); + } @Override protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - resp.setStatus(HttpServletResponse.SC_OK); + checkLoginAndRespond(req, resp); } @Override protected void doPut(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - resp.setStatus(HttpServletResponse.SC_OK); + checkLoginAndRespond(req, resp); } @Override protected void doDelete(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - resp.setStatus(HttpServletResponse.SC_OK); + checkLoginAndRespond(req, resp); + } + + /** + * Checks if request was authenticated for the valid user. + * @param req + * @return + */ + private static boolean isValidLogin(HttpServletRequest req) { + try { + String user = req.getRemoteUser(); + if (req.getAuthType().equals("simple")) { + return user.equals(System.getProperty("user.name")); + } else if(req.getAuthType().equals("kerberos")) { + return user.equals(clientPrincipal); + } + } catch (Exception e) { + LOG.error("Error while checking login", e); + } + return false; + } + + private void checkLoginAndRespond(HttpServletRequest req, + HttpServletResponse resp){ + if(isValidLogin(req)) { + resp.setStatus(HttpServletResponse.SC_OK); + } else { + resp.setStatus(HttpServletResponse.SC_FORBIDDEN); + } + } + } + + /** + * Creates and starts a Jetty server binding at configured port to run the + * Test servlet. + * + * @param protocol "http" or "https" + * @param isSpnego true if SPNEGO is enabled + * @param conf the configuration to use + * @return a created HttpServer2 object + * @throws Exception if unable to create or start a Jetty server + */ + private HttpServer2 createServer(String protocol, boolean isSpnego, + Configuration conf) throws Exception { + String uri; + if ("https".equals(protocol)) { + uri = protocol + "://localhost:8090"; + } else { + uri = protocol + "://localhost:8088"; + } + + HttpServer2.Builder builder = new HttpServer2.Builder().setName("..") + .addEndpoint(new URI(uri)).setConf(conf); + if (isSpnego) { + // Set up server Kerberos credentials. + // Since the server may fall back to simple authentication, + // use ACL to make sure the connection is Kerberos/SPNEGO authenticated. + builder.setSecurityEnabled(true).setUsernameConfKey(principalConfKey) + .setKeytabConfKey(keytabConfKey) + .setACL(new AccessControlList(clientPrincipal)); } - public static boolean isHeaderExist() { - return headerFound; + // if using HTTPS, configure keystore/truststore properties. + if (protocol.equals("https")) { + builder = builder. + keyPassword(sslConf.get("ssl.server.keystore.keypassword")) + .keyStore(sslConf.get("ssl.server.keystore.location"), + sslConf.get("ssl.server.keystore.password"), + sslConf.get("ssl.server.keystore.type", "jks")) + .trustStore(sslConf.get("ssl.server.truststore.location"), + sslConf.get("ssl.server.truststore.password"), + sslConf.get("ssl.server.truststore.type", "jks")); } + Map params = new HashMap<>(); + String apiPackages = "org.apache.hadoop.yarn.service.webapp;" + + "org.apache.hadoop.yarn.webapp"; + params.put("com.sun.jersey.config.property.resourceConfigClass", + "com.sun.jersey.api.core.PackagesResourceConfig"); + params.put("com.sun.jersey.config.property.packages", apiPackages); + + HttpServer2 httpServer2 = builder.build(); + httpServer2.addInternalServlet("TestServlet", "/app/*", + TestServlet.class, params); + httpServer2.start(); + return httpServer2; } @Before public void setUp() throws Exception { - keytabFile = new File(getWorkDir(), "keytab"); - getKdc().createPrincipal(keytabFile, clientPrincipal, server1Principal, - server2Principal); - SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, - testConf); + FileUtil.fullyDelete(BASEDIR); + if (!BASEDIR.mkdirs()) { + throw new Exception("unable to create the base directory for testing"); + } + getKdc().createPrincipal(new File(KerberosTestUtils.getKeytabFile()), + clientPrincipal, server1Principal, server2Principal); UserGroupInformation.setConfiguration(testConf); UserGroupInformation.setShouldRenewImmediatelyForTests(true); - props = new HashMap(); - props.put(Sasl.QOP, QualityOfProtection.AUTHENTICATION.saslQop); - server = new Server(8088); - ((QueuedThreadPool)server.getThreadPool()).setMaxThreads(10); - ServletContextHandler context = new ServletContextHandler(); - context.setContextPath("/app"); - server.setHandler(context); - context.addServlet(new ServletHolder(TestServlet.class), "/*"); - ((ServerConnector)server.getConnectors()[0]).setHost("localhost"); - server.start(); - - List rmServers = new ArrayList(); - rmServers.add("localhost:8088"); - testConf.set("yarn.resourcemanager.webapp.address", - "localhost:8088"); - asc = new ApiServiceClient() { - @Override - List getRMHAWebAddresses(Configuration conf) { - return rmServers; - } - }; - asc.serviceInit(testConf); + setupSSL(BASEDIR); + + testConf.set("yarn.resourcemanager.webapp.address", "localhost:8088"); + testConf.set("yarn.resourcemanager.webapp.https.address", "localhost:8090"); + testConf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY, + AuthenticationFilterInitializer.class.getName()); + } + + private void setupSSL(File base) throws Exception { + keystoresDir = base.getAbsolutePath(); + sslConfDir = KeyStoreTestUtil + .getClasspathDir(TestSecureApiServiceClient.class); + KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, testConf, true); + sslConf = KeyStoreTestUtil.getSslConfig(); + } + + private void configSpnego(Configuration conf) { + conf.set(principalConfKey, KerberosTestUtils.getServerPrincipal()); + conf.set(keytabConfKey, KerberosTestUtils.getKeytabFile()); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + "kerberos"); + conf.set(PREFIX + "type", "kerberos"); + conf.set(PREFIX + "kerberos.keytab", KerberosTestUtils.getKeytabFile()); + conf.set(PREFIX + "kerberos.principal", + KerberosTestUtils.getServerPrincipal()); + conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, + true); + } + + private ApiServiceClient createAsc(Configuration conf) throws Exception { + asc = new ApiServiceClient(); + asc.serviceInit(conf); + return asc; + } + + private void testAppStart(Configuration conf) throws Exception { + try { + String appName = "example-app"; + asc = createAsc(conf); + int result = asc.actionStart(appName); + assertEquals(EXIT_SUCCESS, result); + } catch (IOException | YarnException e) { + fail(); + } + } + + private void testAppStart(String protocol, boolean isSpnego, + final Configuration conf) throws Exception{ + server = createServer(protocol, isSpnego, conf); + if(isSpnego) { + KerberosTestUtils.doAsClient(new Callable() { + @Override + public Void call() throws Exception { + testAppStart(conf); + return null; + } + }); + } else { + testAppStart(conf); + } } @After public void tearDown() throws Exception { + FileUtil.fullyDelete(BASEDIR); server.stop(); + asc.stop(); + } + + @Test + public void testSimpleAuth() throws Exception { + UserGroupInformation.setConfiguration(testConf); + testAppStart("http", false, testConf); } @Test - public void testHttpSpnegoChallenge() throws Exception { - UserGroupInformation.loginUserFromKeytab(clientPrincipal, keytabFile - .getCanonicalPath()); - String challenge = YarnClientUtils.generateToken("localhost"); - assertNotNull(challenge); + public void testSSL() throws Exception { + Configuration sslOnlyConf = new Configuration(testConf); + sslOnlyConf.set("yarn.http.policy", "HTTPS_ONLY"); + sslOnlyConf.addResource(sslConf); + UserGroupInformation.setConfiguration(sslOnlyConf); + testAppStart("https", false, sslOnlyConf); } @Test - public void testAuthorizationHeader() throws Exception { - UserGroupInformation.loginUserFromKeytab(clientPrincipal, keytabFile - .getCanonicalPath()); - String rmAddress = asc.getRMWebAddress(); - if (TestServlet.isHeaderExist()) { - assertEquals(rmAddress, "http://localhost:8088"); - } else { - fail("Did not see Authorization header."); - } + public void testSpnego() throws Exception { + Configuration spengoOnlyConf = new Configuration(testConf); + configSpnego(spengoOnlyConf); + UserGroupInformation.setConfiguration(spengoOnlyConf); + testAppStart("http", true, spengoOnlyConf); } + + @Test + public void testSpnegoWithSSL() throws Exception { + Configuration sslWithSpnegoConf = new Configuration(testConf); + sslWithSpnegoConf.set("yarn.http.policy", "HTTPS_ONLY"); + configSpnego(sslWithSpnegoConf); + sslWithSpnegoConf.addResource(sslConf); + UserGroupInformation.setConfiguration(sslWithSpnegoConf); + testAppStart("https", true, sslWithSpnegoConf); + } + }