diff --git a/common/pom.xml b/common/pom.xml index 8474a87..dd5542e 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -69,20 +69,24 @@ ${jline.version} - org.eclipse.jetty.aggregate - jetty-all - ${jetty.version} - - - javax.servlet - servlet-api - - + javax.servlet + javax.servlet-api + + + org.eclipse.jetty + jetty-rewrite + + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-servlet - org.eclipse.jetty.orbit - javax.servlet - ${javax-servlet.version} + org.eclipse.jetty + jetty-webapp joda-time @@ -129,6 +133,18 @@ servlet-api + javax.servlet.jsp + jsp-api + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + org.slf4j slf4j-log4j12 @@ -149,6 +165,10 @@ servlet-api + org.mortbay.jetty + jetty-util + + org.slf4j slf4j-log4j12 diff --git a/common/src/java/org/apache/hive/http/HttpServer.java b/common/src/java/org/apache/hive/http/HttpServer.java index db5650d..fd3d457 100644 --- a/common/src/java/org/apache/hive/http/HttpServer.java +++ b/common/src/java/org/apache/hive/http/HttpServer.java @@ -42,7 +42,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.authorize.AccessControlList; -import org.apache.hadoop.util.Shell; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.Appender; @@ -54,11 +53,13 @@ import org.eclipse.jetty.rewrite.handler.RewriteHandler; import org.eclipse.jetty.rewrite.handler.RewriteRegexRule; import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.LowResourceMonitor; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.ContextHandler.Context; import org.eclipse.jetty.server.handler.ContextHandlerCollection; -import org.eclipse.jetty.server.nio.SelectChannelConnector; -import org.eclipse.jetty.server.ssl.SslSelectChannelConnector; +import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.servlet.DefaultServlet; import org.eclipse.jetty.servlet.FilterHolder; import org.eclipse.jetty.servlet.FilterMapping; @@ -85,9 +86,9 @@ public static final String ADMINS_ACL = "admins.acl"; private final String name; - private final String appDir; - private final WebAppContext webAppContext; - private final Server webServer; + private String appDir; + private WebAppContext webAppContext; + private Server webServer; /** * Create a status server on the given port. @@ -95,16 +96,7 @@ private HttpServer(final Builder b) throws IOException { this.name = b.name; - webServer = new Server(); - appDir = getWebAppsPath(b.name); - webAppContext = createWebAppContext(b); - - if (b.useSPNEGO) { - // Secure the web server with kerberos - setupSpnegoFilter(b); - } - - initializeWebServer(b); + createWebServer(b); } public static class Builder { @@ -219,7 +211,7 @@ public void stop() throws Exception { } public int getPort() { - return webServer.getConnectors()[0].getLocalPort(); + return ((ServerConnector)(webServer.getConnectors()[0])).getLocalPort(); } /** @@ -345,9 +337,14 @@ void setupSpnegoFilter(Builder b) throws IOException { * Create a channel connector for "http/https" requests */ Connector createChannelConnector(int queueSize, Builder b) { - SelectChannelConnector connector; + ServerConnector connector; + + final HttpConfiguration conf = new HttpConfiguration(); + conf.setRequestHeaderSize(1024*64); + final HttpConnectionFactory http = new HttpConnectionFactory(conf); + if (!b.useSSL) { - connector = new SelectChannelConnector(); + connector = new ServerConnector(webServer, http); } else { SslContextFactory sslContextFactory = new SslContextFactory(); sslContextFactory.setKeyStorePath(b.keyStorePath); @@ -357,15 +354,13 @@ Connector createChannelConnector(int queueSize, Builder b) { sslContextFactory.addExcludeProtocols(excludedSSLProtocols.toArray( new String[excludedSSLProtocols.size()])); sslContextFactory.setKeyStorePassword(b.keyStorePassword); - connector = new SslSelectChannelConnector(sslContextFactory); + connector = new ServerConnector(webServer, sslContextFactory, http); } - connector.setLowResourcesMaxIdleTime(10000); connector.setAcceptQueueSize(queueSize); - connector.setResolveNames(false); - connector.setUseDirectBuffers(false); - connector.setRequestHeaderSize(1024*64); connector.setReuseAddress(true); + connector.setHost(b.host); + connector.setPort(b.port); return connector; } @@ -378,7 +373,7 @@ void setContextAttributes(Context ctx, Map contextAttrs) { } } - void initializeWebServer(Builder b) { + private void createWebServer(final Builder b) throws IOException { // Create the thread pool for the web server to handle HTTP requests QueuedThreadPool threadPool = new QueuedThreadPool(); if (b.maxThreads > 0) { @@ -386,12 +381,26 @@ void initializeWebServer(Builder b) { } threadPool.setDaemon(true); threadPool.setName(b.name + "-web"); - webServer.setThreadPool(threadPool); - // Create the channel connector for the web server - Connector connector = createChannelConnector(threadPool.getMaxThreads(), b); - connector.setHost(b.host); - connector.setPort(b.port); + this.webServer = new Server(threadPool); + this.appDir = getWebAppsPath(b.name); + this.webAppContext = createWebAppContext(b); + + if (b.useSPNEGO) { + // Secure the web server with kerberos + setupSpnegoFilter(b); + } + + initializeWebServer(b, threadPool.getMaxThreads()); + } + + private void initializeWebServer(final Builder b, int queueSize) { + // Set handling for low resource conditions. + final LowResourceMonitor low = new LowResourceMonitor(webServer); + low.setLowResourcesIdleTimeout(10000); + webServer.addBean(low); + + Connector connector = createChannelConnector(queueSize, b); webServer.addConnector(connector); RewriteHandler rwHandler = new RewriteHandler(); diff --git a/hcatalog/pom.xml b/hcatalog/pom.xml index 34de177..9bb82c1 100644 --- a/hcatalog/pom.xml +++ b/hcatalog/pom.xml @@ -71,6 +71,28 @@ ${pig.version} h2 test + + + org.mortbay.jetty + jetty-util + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jsp-api-2.1 + + + org.mortbay.jetty + jsp-2.1 + + + org.mortbay.jetty + servlet-api-2.5 + + diff --git a/hcatalog/webhcat/svr/pom.xml b/hcatalog/webhcat/svr/pom.xml index c5ad387..032f017 100644 --- a/hcatalog/webhcat/svr/pom.xml +++ b/hcatalog/webhcat/svr/pom.xml @@ -45,9 +45,50 @@ hive-hcatalog-core ${project.version} provided + + + org.eclipse.jetty + jetty-runner + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-sslengine + + + org.mortbay.jetty + jetty-util + + + org.mortbay.jetty + jsp-2.1 + + + org.mortbay.jetty + jsp-api-2.1 + + + org.eclipse.jetty + jetty-rewrite + ${jetty.version} + + + org.eclipse.jetty + jetty-server + ${jetty.version} + + + org.eclipse.jetty + jetty-servlet + ${jetty.version} + + com.sun.jersey jersey-core ${jersey.version} @@ -93,11 +134,6 @@ ${jackson.version} - org.eclipse.jetty.aggregate - jetty-all-server - ${jetty.version} - - org.slf4j jul-to-slf4j ${slf4j.version} @@ -107,7 +143,7 @@ hadoop-auth ${hadoop.version} - + org.slf4j slf4j-log4j12 @@ -121,16 +157,42 @@ org.apache.hadoop hadoop-common ${hadoop.version} + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + org.apache.hadoop hadoop-hdfs ${hadoop.version} + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + org.apache.hadoop hadoop-mapreduce-client-core ${hadoop.version} + + + org.mortbay.jetty + jetty-util + + diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java index 5208bf4..3ed3ece 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java @@ -25,6 +25,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.EnumSet; import java.util.HashMap; import org.slf4j.Logger; @@ -43,14 +44,15 @@ import org.eclipse.jetty.rewrite.handler.RewriteHandler; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.HandlerList; import org.eclipse.jetty.servlet.FilterHolder; -import org.eclipse.jetty.servlet.FilterMapping; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.xml.XmlConfiguration; import org.slf4j.bridge.SLF4JBridgeHandler; +import javax.servlet.DispatcherType; import javax.servlet.http.HttpServletRequest; /** @@ -122,7 +124,7 @@ public void run() { checkEnv(); runServer(port); // Currently only print the first port to be consistent with old behavior - port = ArrayUtils.isEmpty(server.getConnectors()) ? -1 : server.getConnectors()[0].getPort(); + port = ArrayUtils.isEmpty(server.getConnectors()) ? -1 : ((ServerConnector)(server.getConnectors()[0])).getLocalPort(); System.out.println("templeton: listening on port " + port); LOG.info("Templeton listening on port " + port); @@ -185,6 +187,7 @@ public Server runServer(int port) // Add the Auth filter FilterHolder fHolder = makeAuthFilter(); + EnumSet dispatches = EnumSet.of(DispatcherType.REQUEST); /* * We add filters for each of the URIs supported by templeton. @@ -193,28 +196,18 @@ public Server runServer(int port) * This is because mapreduce does not use secure credentials for * callbacks. So jetty would fail the request as unauthorized. */ - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/ddl/*", - FilterMapping.REQUEST); - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/pig/*", - FilterMapping.REQUEST); - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/hive/*", - FilterMapping.REQUEST); - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/sqoop/*", - FilterMapping.REQUEST); - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/queue/*", - FilterMapping.REQUEST); - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/jobs/*", - FilterMapping.REQUEST); - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/mapreduce/*", - FilterMapping.REQUEST); - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/status/*", - FilterMapping.REQUEST); - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/version/*", - FilterMapping.REQUEST); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/ddl/*", dispatches); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/pig/*", dispatches); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/hive/*", dispatches); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/sqoop/*", dispatches); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/queue/*", dispatches); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/jobs/*", dispatches); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/mapreduce/*", dispatches); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/status/*", dispatches); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/version/*", dispatches); if (conf.getBoolean(AppConfig.XSRF_FILTER_ENABLED, false)){ - root.addFilter(makeXSRFFilter(), "/" + SERVLET_PATH + "/*", - FilterMapping.REQUEST); + root.addFilter(makeXSRFFilter(), "/" + SERVLET_PATH + "/*", dispatches); LOG.debug("XSRF filter enabled"); } else { LOG.warn("XSRF filter disabled"); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java index 22e5ee8..4a4fef0 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java @@ -76,7 +76,7 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.eclipse.jetty.server.ssl.SslSocketConnector; +import org.eclipse.jetty.util.ssl.SslContextFactory; import org.joda.time.DateTime; import org.json.JSONException; import org.json.JSONObject; @@ -378,7 +378,7 @@ public Void call() throws Exception { LlapTezUtils.class, // llap-tez LlapInputFormat.class, // llap-server HiveInputFormat.class, // hive-exec - SslSocketConnector.class, // hive-common (https deps) + SslContextFactory.class, // hive-common (https deps) RegistryUtils.ServiceRecordMarshal.class, // ZK registry // log4j2 com.lmax.disruptor.RingBuffer.class, // disruptor diff --git a/pom.xml b/pom.xml index 4dd5262..0320ab0 100644 --- a/pom.xml +++ b/pom.xml @@ -159,10 +159,10 @@ 2.3.4 2.3.1 0.3.2 - 3.0.0.v201112011016 + 3.1.0 5.5.1 3.0.1 - 7.6.0.v20120127 + 9.3.8.v20160314 1.14 2.22.2 @@ -610,11 +610,31 @@ ${jackson.version} - org.eclipse.jetty.aggregate - jetty-all-server + org.eclipse.jetty + jetty-rewrite ${jetty.version} + org.eclipse.jetty + jetty-server + ${jetty.version} + + + org.eclipse.jetty + jetty-servlet + ${jetty.version} + + + org.eclipse.jetty + jetty-webapp + ${jetty.version} + + + javax.servlet + javax.servlet-api + ${javax-servlet.version} + + org.datanucleus datanucleus-api-jdo ${datanucleus-api-jdo.version} diff --git a/service/pom.xml b/service/pom.xml index 9306739..7c0fc4c 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -53,6 +53,16 @@ org.apache.hive hive-llap-server ${project.version} + + + javax.servlet + servlet-api + + + org.mortbay.jetty + servlet-api-2.5 + + @@ -70,8 +80,8 @@ net.sf.jpam jpam ${jpam.version} - - + + org.slf4j slf4j-log4j12 @@ -81,6 +91,25 @@ + + org.eclipse.jetty + jetty-server + ${jetty.version} + + + org.eclipse.jetty + jetty-servlet + ${jetty.version} + + + org.eclipse.jetty + jetty-runner + ${jetty.version} + + + javax.servlet + javax.servlet-api + commons-lang @@ -88,19 +117,26 @@ ${commons-lang.version} - org.eclipse.jetty.aggregate - jetty-all - ${jetty.version} - - tomcat jasper-compiler ${jasper.version} + + + javax.servlet + servlet-api + + tomcat jasper-runtime ${jasper.version} + + + javax.servlet + servlet-api + + org.apache.thrift @@ -127,12 +163,60 @@ hadoop-common ${hadoop.version} true + + + commons-collections + commons-collections + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + org.apache.hadoop hadoop-mapreduce-client-core ${hadoop.version} true + + + javax.servlet + servlet-api + + + org.mortbay.jetty + jetty-util + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + org.jamon diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java index ebec165..c4d4e02 100644 --- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java +++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java @@ -38,8 +38,10 @@ import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TProtocolFactory; import org.apache.thrift.server.TServlet; -import org.eclipse.jetty.server.nio.SelectChannelConnector; -import org.eclipse.jetty.server.ssl.SslSelectChannelConnector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.servlet.FilterMapping; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; @@ -63,9 +65,6 @@ public ThriftHttpCLIService(CLIService cliService, Runnable oomHook) { @Override public void run() { try { - // HTTP Server - httpServer = new org.eclipse.jetty.server.Server(); - // Server thread pool // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests String threadPoolName = "HiveServer2-HttpHandler-Pool"; @@ -73,19 +72,26 @@ public void run() { maxWorkerThreads, workerKeepAliveTime, TimeUnit.SECONDS, new SynchronousQueue(), new ThreadFactoryWithGarbageCleanup(threadPoolName), oomHook); ExecutorThreadPool threadPool = new ExecutorThreadPool(executorService); - httpServer.setThreadPool(threadPool); - // Connector configs - SelectChannelConnector connector = new SelectChannelConnector(); + // HTTP Server + httpServer = new Server(threadPool); + + + ServerConnector connector; + + final HttpConfiguration conf = new HttpConfiguration(); // Configure header size int requestHeaderSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_REQUEST_HEADER_SIZE); int responseHeaderSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_RESPONSE_HEADER_SIZE); - connector.setRequestHeaderSize(requestHeaderSize); - connector.setResponseHeaderSize(responseHeaderSize); + conf.setRequestHeaderSize(requestHeaderSize); + conf.setResponseHeaderSize(responseHeaderSize); + final HttpConnectionFactory http = new HttpConnectionFactory(conf); + boolean useSsl = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL); String schemeName = useSsl ? "https" : "http"; + // Change connector if SSL is used if (useSsl) { String keyStorePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH).trim(); @@ -103,14 +109,17 @@ public void run() { Arrays.toString(sslContextFactory.getExcludeProtocols())); sslContextFactory.setKeyStorePath(keyStorePath); sslContextFactory.setKeyStorePassword(keyStorePassword); - connector = new SslSelectChannelConnector(sslContextFactory); + connector = new ServerConnector(httpServer, sslContextFactory, http); + } else { + connector = new ServerConnector(httpServer, http); } + connector.setPort(portNum); // Linux:yes, Windows:no connector.setReuseAddress(true); int maxIdleTime = (int) hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME, TimeUnit.MILLISECONDS); - connector.setMaxIdleTime(maxIdleTime); + connector.setIdleTimeout(maxIdleTime); httpServer.addConnector(connector); diff --git a/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java b/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java index 4d50dd9..663a266 100644 --- a/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java +++ b/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java @@ -142,7 +142,7 @@ public void testConfStrippedFromWebUI() throws Exception { private String getURLResponseAsString(String baseURL) throws IOException { URL url = new URL(baseURL); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + Assert.assertEquals("Got an HTTP response code other thank OK.", HttpURLConnection.HTTP_OK, conn.getResponseCode()); StringWriter writer = new StringWriter(); IOUtils.copy(conn.getInputStream(), writer, "UTF-8"); return writer.toString();