diff --git common/pom.xml common/pom.xml index e6722ba..41ba568 100644 --- common/pom.xml +++ common/pom.xml @@ -86,6 +86,10 @@ org.eclipse.jetty + jetty-servlets + + + org.eclipse.jetty jetty-webapp diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index a8bdefd..3c1d54c 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2454,6 +2454,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "Request header size in bytes, when using HTTP transport mode. Jetty defaults used."), HIVE_SERVER2_THRIFT_HTTP_RESPONSE_HEADER_SIZE("hive.server2.thrift.http.response.header.size", 6*1024, "Response header size in bytes, when using HTTP transport mode. Jetty defaults used."), + HIVE_SERVER2_THRIFT_HTTP_COMPRESSION_LEVEL("hive.server2.thrift.http.compression.level", 3, + "The compression level used by the thrift http service via Jetty compression support"), // Cookie based authentication when using HTTP Transport HIVE_SERVER2_THRIFT_HTTP_COOKIE_AUTH_ENABLED("hive.server2.thrift.http.cookie.auth.enabled", true, diff --git common/src/java/org/apache/hive/http/HttpServer.java common/src/java/org/apache/hive/http/HttpServer.java index 7368a91..079ff46 100644 --- common/src/java/org/apache/hive/http/HttpServer.java +++ common/src/java/org/apache/hive/http/HttpServer.java @@ -33,6 +33,7 @@ import javax.servlet.http.HttpServletResponse; import com.google.common.base.Preconditions; + import org.apache.commons.math3.util.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -66,6 +67,7 @@ import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHandler; import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.servlets.GzipFilter; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.eclipse.jetty.webapp.WebAppContext; @@ -73,6 +75,7 @@ import com.google.common.base.Splitter; import com.google.common.base.Strings; import com.google.common.collect.Sets; + import org.slf4j.LoggerFactory; /** @@ -112,6 +115,7 @@ private HttpServer(final Builder b) throws IOException { private String spnegoKeytab; private boolean useSPNEGO; private boolean useSSL; + private int compressionLevel = -1; private String contextRootRewriteTarget = "/index.html"; private final List>> servlets = new LinkedList>>(); @@ -170,6 +174,11 @@ public Builder setUseSSL(boolean useSSL) { return this; } + public Builder setCompressionLevel(int compressionLevel) { + this.compressionLevel = Math.max(-1, Math.min(9, compressionLevel)); + return this; + } + public Builder setUseSPNEGO(boolean useSPNEGO) { this.useSPNEGO = useSPNEGO; return this; @@ -334,6 +343,24 @@ void setupSpnegoFilter(Builder b) throws IOException { handler.addFilterWithMapping( holder, "/*", FilterMapping.ALL); } + + /** + * setup Gzip compression (GzipFilter). + */ + void setupGzipFilter(Builder b) throws IOException { + Map params = new HashMap(); + params.put("mimeTypes", + "text/html,text/plain,text/xml,application/xhtml+xml,text/css,application/javascript," + + "image/svg+xml,application/json"); + params.put("deflateCompressionLevel", String.valueOf(b.compressionLevel)); + params.put("methods", "GET,POST"); + FilterHolder holder = new FilterHolder(); + holder.setClassName(GzipFilter.class.getName()); + holder.setInitParameters(params); + + ServletHandler handler = webAppContext.getServletHandler(); + handler.addFilterWithMapping(holder, "/*", FilterMapping.ALL); + } /** * Create a channel connector for "http/https" requests @@ -393,6 +420,11 @@ private void createWebServer(final Builder b) throws IOException { setupSpnegoFilter(b); } + if (b.compressionLevel > 0) { + // allow HTTP compression with gzip + setupGzipFilter(b); + } + initializeWebServer(b, threadPool.getMaxThreads()); } diff --git pom.xml pom.xml index 85f1222..b626919 100644 --- pom.xml +++ pom.xml @@ -626,6 +626,11 @@ org.eclipse.jetty + jetty-servlets + ${jetty.version} + + + org.eclipse.jetty jetty-webapp ${jetty.version} diff --git service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java index c4d4e02..e0f7c9b 100644 --- service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java +++ service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java @@ -19,6 +19,8 @@ package org.apache.hive.service.cli.thrift; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.TimeUnit; @@ -42,9 +44,12 @@ import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.servlet.FilterHolder; import org.eclipse.jetty.servlet.FilterMapping; import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHandler; import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.servlets.GzipFilter; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.ExecutorThreadPool; @@ -148,6 +153,20 @@ public void run() { } else { LOG.warn("XSRF filter disabled"); } + + if (hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COMPRESSION_LEVEL) > 0) { + final int compressionLevel = + hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COMPRESSION_LEVEL); + Map params = new HashMap(); + params.put("mimeTypes", "application/x-thrift"); + params.put("deflateCompressionLevel", String.valueOf(compressionLevel)); + params.put("methods", "GET,POST"); + FilterHolder holder = new FilterHolder(); + holder.setClassName(GzipFilter.class.getName()); + holder.setInitParameters(params); + ServletHandler handler = context.getServletHandler(); + handler.addFilterWithMapping(holder, "/*", FilterMapping.ALL); + } String httpPath = getHttpPath(hiveConf .getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH)); diff --git service/src/java/org/apache/hive/service/server/HiveServer2.java service/src/java/org/apache/hive/service/server/HiveServer2.java index e5f4491..5bfbcb0 100644 --- service/src/java/org/apache/hive/service/server/HiveServer2.java +++ service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -222,6 +222,8 @@ public void run() { builder.setSPNEGOKeytab(spnegoKeytab); builder.setUseSPNEGO(true); } + builder.setCompressionLevel(hiveConf + .getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COMPRESSION_LEVEL)); builder.addServlet("llap", LlapServlet.class); builder.setContextRootRewriteTarget("/hiveserver2.jsp"); webServer = builder.build();