From f1ab270321ed942e8fe8e5f7a1edec2923b613f0 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Sun, 26 Feb 2017 00:54:13 -0600 Subject: [PATCH] HIVE-16049 update to jdk8+, jetty 9.3, and latest apache parent pom. * Update to apache parent pom v18 * switch to using user properties to specify source/target jdk. * use source/target jdk of 1.8 * update maven-compiler-plugin to latest (3.6.1) * update Jetty to stable jdk8+ version (9.3.8.v20160314) * Remove extraneous non-compatible sources of javax.servlet --- .travis.yml | 2 +- common/pom.xml | 44 +++++++--- .../src/java/org/apache/hive/http/HttpServer.java | 57 ++++++++----- hcatalog/build.properties | 2 +- hcatalog/webhcat/svr/pom.xml | 5 +- .../org/apache/hive/hcatalog/templeton/Main.java | 37 ++++----- .../hadoop/hive/llap/cli/LlapServiceDriver.java | 4 +- pom.xml | 40 ++++++--- service/pom.xml | 94 ++++++++++++++++++++-- .../service/cli/thrift/ThriftHttpCLIService.java | 33 +++++--- .../hive/service/server/TestHS2HttpServer.java | 2 +- spark-client/pom.xml | 2 +- 12 files changed, 229 insertions(+), 93 deletions(-) diff --git a/.travis.yml b/.travis.yml index d0e1568..d13b996 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,7 +35,7 @@ cache: env: MAVEN_SKIP_RC=true - MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512M" + MAVEN_OPTS="-Xmx2g" # workaround added: https://github.com/travis-ci/travis-ci/issues/4629 before_install: diff --git a/common/pom.xml b/common/pom.xml index 8474a87..dd5542e 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -69,20 +69,24 @@ ${jline.version} - org.eclipse.jetty.aggregate - jetty-all - ${jetty.version} - - - javax.servlet - servlet-api - - + javax.servlet + javax.servlet-api + + + org.eclipse.jetty + jetty-rewrite + + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-servlet - org.eclipse.jetty.orbit - javax.servlet - ${javax-servlet.version} + org.eclipse.jetty + jetty-webapp joda-time @@ -129,6 +133,18 @@ servlet-api + javax.servlet.jsp + jsp-api + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + org.slf4j slf4j-log4j12 @@ -149,6 +165,10 @@ servlet-api + org.mortbay.jetty + jetty-util + + org.slf4j slf4j-log4j12 diff --git a/common/src/java/org/apache/hive/http/HttpServer.java b/common/src/java/org/apache/hive/http/HttpServer.java index db5650d..85b5529 100644 --- a/common/src/java/org/apache/hive/http/HttpServer.java +++ b/common/src/java/org/apache/hive/http/HttpServer.java @@ -54,11 +54,13 @@ import org.eclipse.jetty.rewrite.handler.RewriteHandler; import org.eclipse.jetty.rewrite.handler.RewriteRegexRule; import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.LowResourceMonitor; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.ContextHandler.Context; import org.eclipse.jetty.server.handler.ContextHandlerCollection; -import org.eclipse.jetty.server.nio.SelectChannelConnector; -import org.eclipse.jetty.server.ssl.SslSelectChannelConnector; +import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.servlet.DefaultServlet; import org.eclipse.jetty.servlet.FilterHolder; import org.eclipse.jetty.servlet.FilterMapping; @@ -95,7 +97,15 @@ private HttpServer(final Builder b) throws IOException { this.name = b.name; - webServer = new Server(); + // Create the thread pool for the web server to handle HTTP requests + QueuedThreadPool threadPool = new QueuedThreadPool(); + if (b.maxThreads > 0) { + threadPool.setMaxThreads(b.maxThreads); + } + threadPool.setDaemon(true); + threadPool.setName(b.name + "-web"); + + webServer = new Server(threadPool); appDir = getWebAppsPath(b.name); webAppContext = createWebAppContext(b); @@ -219,7 +229,7 @@ public void stop() throws Exception { } public int getPort() { - return webServer.getConnectors()[0].getLocalPort(); + return ((ServerConnector)(webServer.getConnectors()[0])).getLocalPort(); } /** @@ -345,9 +355,14 @@ void setupSpnegoFilter(Builder b) throws IOException { * Create a channel connector for "http/https" requests */ Connector createChannelConnector(int queueSize, Builder b) { - SelectChannelConnector connector; + ServerConnector connector; + + final HttpConfiguration conf = new HttpConfiguration(); + conf.setRequestHeaderSize(1024*64); + final HttpConnectionFactory http = new HttpConnectionFactory(conf); + if (!b.useSSL) { - connector = new SelectChannelConnector(); + connector = new ServerConnector(webServer, http); } else { SslContextFactory sslContextFactory = new SslContextFactory(); sslContextFactory.setKeyStorePath(b.keyStorePath); @@ -357,15 +372,17 @@ Connector createChannelConnector(int queueSize, Builder b) { sslContextFactory.addExcludeProtocols(excludedSSLProtocols.toArray( new String[excludedSSLProtocols.size()])); sslContextFactory.setKeyStorePassword(b.keyStorePassword); - connector = new SslSelectChannelConnector(sslContextFactory); + connector = new ServerConnector(webServer, sslContextFactory, http); } - connector.setLowResourcesMaxIdleTime(10000); connector.setAcceptQueueSize(queueSize); - connector.setResolveNames(false); - connector.setUseDirectBuffers(false); - connector.setRequestHeaderSize(1024*64); + // In Jetty 9 I believe this isn't needed, since the default is not to resolve names + //connector.setResolveNames(false); + // In Jetty 9 no direct buffers + //connector.setUseDirectBuffers(false); connector.setReuseAddress(true); + connector.setHost(b.host); + connector.setPort(b.port); return connector; } @@ -379,19 +396,15 @@ void setContextAttributes(Context ctx, Map contextAttrs) { } void initializeWebServer(Builder b) { - // Create the thread pool for the web server to handle HTTP requests - QueuedThreadPool threadPool = new QueuedThreadPool(); - if (b.maxThreads > 0) { - threadPool.setMaxThreads(b.maxThreads); - } - threadPool.setDaemon(true); - threadPool.setName(b.name + "-web"); - webServer.setThreadPool(threadPool); + // Set handling for low resource conditions. + final LowResourceMonitor low = new LowResourceMonitor(webServer); + low.setLowResourcesIdleTimeout(10000); + webServer.addBean(low); // Create the channel connector for the web server - Connector connector = createChannelConnector(threadPool.getMaxThreads(), b); - connector.setHost(b.host); - connector.setPort(b.port); + final int maxThreads = b.maxThreads; + // the default used in QueuedThreadPool for Jetty 9.3 when maxThreads is !> 0 is 200 + Connector connector = createChannelConnector(maxThreads > 0 ? maxThreads : 200, b); webServer.addConnector(connector); RewriteHandler rwHandler = new RewriteHandler(); diff --git a/hcatalog/build.properties b/hcatalog/build.properties index dea1a44..3767cf3 100644 --- a/hcatalog/build.properties +++ b/hcatalog/build.properties @@ -49,7 +49,7 @@ clover.report.dir=${build.dir}/test/clover/reports clover.pdf.report.dir=${build.dir}/test/clover/pdf/reports # junit jvm args -junit.jvm.args=-XX:-UseSplitVerifier -XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=128M +junit.jvm.args=-XX:-UseSplitVerifier -XX:+CMSClassUnloadingEnabled apache-rat.version=0.8 diff --git a/hcatalog/webhcat/svr/pom.xml b/hcatalog/webhcat/svr/pom.xml index c5ad387..8ad066a 100644 --- a/hcatalog/webhcat/svr/pom.xml +++ b/hcatalog/webhcat/svr/pom.xml @@ -93,9 +93,8 @@ ${jackson.version} - org.eclipse.jetty.aggregate - jetty-all-server - ${jetty.version} + org.eclipse.jetty + jetty-server org.slf4j diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java index 5208bf4..3ed3ece 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java @@ -25,6 +25,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.EnumSet; import java.util.HashMap; import org.slf4j.Logger; @@ -43,14 +44,15 @@ import org.eclipse.jetty.rewrite.handler.RewriteHandler; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.HandlerList; import org.eclipse.jetty.servlet.FilterHolder; -import org.eclipse.jetty.servlet.FilterMapping; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.xml.XmlConfiguration; import org.slf4j.bridge.SLF4JBridgeHandler; +import javax.servlet.DispatcherType; import javax.servlet.http.HttpServletRequest; /** @@ -122,7 +124,7 @@ public void run() { checkEnv(); runServer(port); // Currently only print the first port to be consistent with old behavior - port = ArrayUtils.isEmpty(server.getConnectors()) ? -1 : server.getConnectors()[0].getPort(); + port = ArrayUtils.isEmpty(server.getConnectors()) ? -1 : ((ServerConnector)(server.getConnectors()[0])).getLocalPort(); System.out.println("templeton: listening on port " + port); LOG.info("Templeton listening on port " + port); @@ -185,6 +187,7 @@ public Server runServer(int port) // Add the Auth filter FilterHolder fHolder = makeAuthFilter(); + EnumSet dispatches = EnumSet.of(DispatcherType.REQUEST); /* * We add filters for each of the URIs supported by templeton. @@ -193,28 +196,18 @@ public Server runServer(int port) * This is because mapreduce does not use secure credentials for * callbacks. So jetty would fail the request as unauthorized. */ - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/ddl/*", - FilterMapping.REQUEST); - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/pig/*", - FilterMapping.REQUEST); - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/hive/*", - FilterMapping.REQUEST); - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/sqoop/*", - FilterMapping.REQUEST); - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/queue/*", - FilterMapping.REQUEST); - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/jobs/*", - FilterMapping.REQUEST); - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/mapreduce/*", - FilterMapping.REQUEST); - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/status/*", - FilterMapping.REQUEST); - root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/version/*", - FilterMapping.REQUEST); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/ddl/*", dispatches); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/pig/*", dispatches); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/hive/*", dispatches); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/sqoop/*", dispatches); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/queue/*", dispatches); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/jobs/*", dispatches); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/mapreduce/*", dispatches); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/status/*", dispatches); + root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/version/*", dispatches); if (conf.getBoolean(AppConfig.XSRF_FILTER_ENABLED, false)){ - root.addFilter(makeXSRFFilter(), "/" + SERVLET_PATH + "/*", - FilterMapping.REQUEST); + root.addFilter(makeXSRFFilter(), "/" + SERVLET_PATH + "/*", dispatches); LOG.debug("XSRF filter enabled"); } else { LOG.warn("XSRF filter disabled"); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java index e8517ab..cae01f7 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java @@ -76,7 +76,7 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.eclipse.jetty.server.ssl.SslSocketConnector; +import org.eclipse.jetty.util.ssl.SslContextFactory; import org.joda.time.DateTime; import org.json.JSONException; import org.json.JSONObject; @@ -378,7 +378,7 @@ public Void call() throws Exception { LlapTezUtils.class, // llap-tez LlapInputFormat.class, // llap-server HiveInputFormat.class, // hive-exec - SslSocketConnector.class, // hive-common (https deps) + SslContextFactory.class, // hive-common (https deps) RegistryUtils.ServiceRecordMarshal.class, // ZK registry // log4j2 com.lmax.disruptor.RingBuffer.class, // disruptor diff --git a/pom.xml b/pom.xml index bfa66a1..de21f41 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ org.apache apache - 14 + 18 org.apache.hive hive @@ -65,6 +65,8 @@ UTF-8 + 1.8 + 1.8 false ${settings.localRepository} . @@ -90,11 +92,11 @@ 1.0b3 3.3.0-release - -Xmx1024m -XX:MaxPermSize=256M + -Xmx1024m 1.7 2.3 2.12.1 - 3.1 + 3.6.1 1.3.1 2.4 2.4 @@ -157,10 +159,10 @@ 2.3.4 2.3.1 0.3.2 - 3.0.0.v201112011016 + 3.1.0 5.5.1 3.0.1 - 7.6.0.v20120127 + 9.3.8.v20160314 1.14 2.22.2 @@ -608,11 +610,31 @@ ${jackson.version} - org.eclipse.jetty.aggregate - jetty-all-server + org.eclipse.jetty + jetty-rewrite ${jetty.version} + org.eclipse.jetty + jetty-server + ${jetty.version} + + + org.eclipse.jetty + jetty-servlet + ${jetty.version} + + + org.eclipse.jetty + jetty-webapp + ${jetty.version} + + + javax.servlet + javax.servlet-api + ${javax-servlet.version} + + org.datanucleus datanucleus-api-jdo ${datanucleus-api-jdo.version} @@ -803,10 +825,6 @@ org.apache.maven.plugins maven-compiler-plugin ${maven.compiler.plugin.version} - - 1.7 - 1.7 - org.apache.maven.plugins diff --git a/service/pom.xml b/service/pom.xml index 9306739..2126f77 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -53,6 +53,16 @@ org.apache.hive hive-llap-server ${project.version} + + + javax.servlet + servlet-api + + + org.mortbay.jetty + servlet-api-2.5 + + @@ -81,6 +91,25 @@ + + org.eclipse.jetty + jetty-server + ${jetty.version} + + + org.eclipse.jetty + jetty-servlet + ${jetty.version} + + + org.eclipse.jetty + jetty-runner + ${jetty.version} + + + javax.servlet + javax.servlet-api + commons-lang @@ -88,19 +117,26 @@ ${commons-lang.version} - org.eclipse.jetty.aggregate - jetty-all - ${jetty.version} - - tomcat jasper-compiler ${jasper.version} + + + javax.servlet + servlet-api + + tomcat jasper-runtime ${jasper.version} + + + javax.servlet + servlet-api + + org.apache.thrift @@ -127,12 +163,60 @@ hadoop-common ${hadoop.version} true + + + commons-collections + commons-collections + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + org.apache.hadoop hadoop-mapreduce-client-core ${hadoop.version} true + + + javax.servlet + servlet-api + + + org.mortbay.jetty + jetty-util + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + org.jamon diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java index ebec165..c4d4e02 100644 --- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java +++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java @@ -38,8 +38,10 @@ import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TProtocolFactory; import org.apache.thrift.server.TServlet; -import org.eclipse.jetty.server.nio.SelectChannelConnector; -import org.eclipse.jetty.server.ssl.SslSelectChannelConnector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.servlet.FilterMapping; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; @@ -63,9 +65,6 @@ public ThriftHttpCLIService(CLIService cliService, Runnable oomHook) { @Override public void run() { try { - // HTTP Server - httpServer = new org.eclipse.jetty.server.Server(); - // Server thread pool // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests String threadPoolName = "HiveServer2-HttpHandler-Pool"; @@ -73,19 +72,26 @@ public void run() { maxWorkerThreads, workerKeepAliveTime, TimeUnit.SECONDS, new SynchronousQueue(), new ThreadFactoryWithGarbageCleanup(threadPoolName), oomHook); ExecutorThreadPool threadPool = new ExecutorThreadPool(executorService); - httpServer.setThreadPool(threadPool); - // Connector configs - SelectChannelConnector connector = new SelectChannelConnector(); + // HTTP Server + httpServer = new Server(threadPool); + + + ServerConnector connector; + + final HttpConfiguration conf = new HttpConfiguration(); // Configure header size int requestHeaderSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_REQUEST_HEADER_SIZE); int responseHeaderSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_RESPONSE_HEADER_SIZE); - connector.setRequestHeaderSize(requestHeaderSize); - connector.setResponseHeaderSize(responseHeaderSize); + conf.setRequestHeaderSize(requestHeaderSize); + conf.setResponseHeaderSize(responseHeaderSize); + final HttpConnectionFactory http = new HttpConnectionFactory(conf); + boolean useSsl = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL); String schemeName = useSsl ? "https" : "http"; + // Change connector if SSL is used if (useSsl) { String keyStorePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH).trim(); @@ -103,14 +109,17 @@ public void run() { Arrays.toString(sslContextFactory.getExcludeProtocols())); sslContextFactory.setKeyStorePath(keyStorePath); sslContextFactory.setKeyStorePassword(keyStorePassword); - connector = new SslSelectChannelConnector(sslContextFactory); + connector = new ServerConnector(httpServer, sslContextFactory, http); + } else { + connector = new ServerConnector(httpServer, http); } + connector.setPort(portNum); // Linux:yes, Windows:no connector.setReuseAddress(true); int maxIdleTime = (int) hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME, TimeUnit.MILLISECONDS); - connector.setMaxIdleTime(maxIdleTime); + connector.setIdleTimeout(maxIdleTime); httpServer.addConnector(connector); diff --git a/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java b/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java index 351cfc1..f2cb71f 100644 --- a/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java +++ b/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java @@ -139,7 +139,7 @@ public void testConfStrippedFromWebUI() throws Exception { private String getURLResponseAsString(String baseURL) throws IOException { URL url = new URL(baseURL); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + Assert.assertEquals("Got an HTTP response code other thank OK.", HttpURLConnection.HTTP_OK, conn.getResponseCode()); StringWriter writer = new StringWriter(); IOUtils.copy(conn.getInputStream(), writer, "UTF-8"); return writer.toString(); diff --git a/spark-client/pom.xml b/spark-client/pom.xml index effc13b..6faa627 100644 --- a/spark-client/pom.xml +++ b/spark-client/pom.xml @@ -184,7 +184,7 @@ ${test.redirectToFile} ${test.redirectToFile} - -Xmx4096m -XX:MaxPermSize=512m + -Xmx4096m -- 2.7.2