commit baffc1c099a87df2af3403f1a4d6528bc7eea2e6 Author: Eric Yang Date: Thu Oct 11 20:33:42 2018 -0400 YARN-8869. Fixed Kerberos authorization header bug for YARN service REST API. Contributed by Eric Yang diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java index ca6cc50..1692361 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java @@ -144,7 +144,7 @@ public String run() throws Exception { /** * Calculate Resource Manager address base on working REST API. */ - private String getRMWebAddress() { + String getRMWebAddress() { Configuration conf = getConfig(); String scheme = "http://"; String path = "/app/v1/services/version"; @@ -156,8 +156,7 @@ private String getRMWebAddress() { .get("yarn.resourcemanager.webapp.https.address"); } boolean useKerberos = UserGroupInformation.isSecurityEnabled(); - List rmServers = RMHAUtils - .getRMHAWebappAddresses(new YarnConfiguration(conf)); + List rmServers = getRMHAWebAddresses(conf); for (String host : rmServers) { try { Client client = Client.create(); @@ -175,16 +174,16 @@ private String getRMWebAddress() { LOG.debug("Fail to resolve username: {}", e); } } - WebResource webResource = client - .resource(sb.toString()); + Builder builder = client + .resource(sb.toString()).type(MediaType.APPLICATION_JSON);; if (useKerberos) { String[] server = host.split(":"); String challenge = generateToken(server[0]); - webResource.header(HttpHeaders.AUTHORIZATION, "Negotiate " + + builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " + challenge); LOG.debug("Authorization: Negotiate {}", challenge); } - ClientResponse test = webResource.get(ClientResponse.class); + ClientResponse test = builder.get(ClientResponse.class); if (test.getStatus() == 200) { rmAddress = host; break; @@ -197,6 +196,11 @@ private String getRMWebAddress() { return scheme+rmAddress; } + List getRMHAWebAddresses(Configuration conf) { + return RMHAUtils + .getRMHAWebappAddresses(new YarnConfiguration(conf)); + } + /** * Compute active resource manager API service location. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java index 4f3b461..cedfd02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java @@ -21,11 +21,19 @@ import static org.junit.Assert.*; import java.io.File; +import java.io.IOException; import javax.security.sasl.Sasl; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.util.Map; +import java.util.ArrayList; +import java.util.Enumeration; import java.util.HashMap; +import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.minikdc.KerberosSecurityTestcase; @@ -33,6 +41,13 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.SaslRpcServer.QualityOfProtection; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.log4j.Logger; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.thread.QueuedThreadPool; +import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -58,6 +73,55 @@ private Configuration conf = new Configuration(); private Map props; + private static Server server; + private static ApiServiceClient asc; + private static Logger LOG = Logger.getLogger(TestSecureApiServiceClient.class); + + /** + * A mocked version of API Service for testing purpose. + * + */ + @SuppressWarnings("serial") + public static class TestServlet extends HttpServlet { + + public static boolean headerFound = false; + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + Enumeration headers = req.getHeaderNames(); + while(headers.hasMoreElements()) { + String header = headers.nextElement(); + LOG.info(header); + } + if (req.getHeader("Authorization")!=null) { + headerFound = true; + resp.setStatus(HttpServletResponse.SC_OK); + } else { + headerFound = false; + resp.setStatus(HttpServletResponse.SC_NOT_FOUND); + } + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + resp.setStatus(HttpServletResponse.SC_OK); + } + + @Override + protected void doPut(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + resp.setStatus(HttpServletResponse.SC_OK); + } + + @Override + protected void doDelete(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + resp.setStatus(HttpServletResponse.SC_OK); + } + + } @Before public void setUp() throws Exception { @@ -69,6 +133,26 @@ public void setUp() throws Exception { UserGroupInformation.setShouldRenewImmediatelyForTests(true); props = new HashMap(); props.put(Sasl.QOP, QualityOfProtection.AUTHENTICATION.saslQop); + server = new Server(8088); + ((QueuedThreadPool)server.getThreadPool()).setMaxThreads(10); + ServletContextHandler context = new ServletContextHandler(); + context.setContextPath("/app"); + server.setHandler(context); + context.addServlet(new ServletHolder(TestServlet.class), "/*"); + ((ServerConnector)server.getConnectors()[0]).setHost("localhost"); + server.start(); + + Configuration conf = new Configuration(); + conf.set("yarn.resourcemanager.webapp.address", + "localhost:8088"); + asc = new ApiServiceClient(); + asc.serviceInit(conf); + + } + + @After + public void tearDown() throws Exception { + server.stop(); } @Test @@ -80,4 +164,24 @@ public void testHttpSpnegoChallenge() throws Exception { assertNotNull(challenge); } + @Test + public void testAuthorizationHeader() throws Exception { + List rmServers = new ArrayList(); + rmServers.add("localhost:8088"); + UserGroupInformation.loginUserFromKeytab(clientPrincipal, keytabFile + .getCanonicalPath()); + ApiServiceClient asc = new ApiServiceClient() { + @Override + List getRMHAWebAddresses(Configuration conf) { + return rmServers; + } + }; + asc.serviceInit(conf); + String rmAddress = asc.getRMWebAddress(); + if (TestServlet.headerFound) { + assertEquals(rmAddress, "http://localhost:8088"); + } else { + fail("Did not see Authorization header."); + } + } }