diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index e2bd38b..41463db 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2516,8 +2516,11 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "excessive threads are killed after this time interval."), // Configuration for async thread pool in SessionManager + HIVE_SERVER2_ASYNC_EXEC_THREADS_MIN("hive.server2.async.exec.threads.min", -1, + "The minimum number of threads in the async thread pool for HiveServer2. Default -1 meaning " + + "the same as hive.server2.async.exec.threads (fixed sized thread pool)."), HIVE_SERVER2_ASYNC_EXEC_THREADS("hive.server2.async.exec.threads", 100, - "Number of threads in the async thread pool for HiveServer2"), + "The maxmium number of threads in the async thread pool for HiveServer2."), HIVE_SERVER2_ASYNC_EXEC_SHUTDOWN_TIMEOUT("hive.server2.async.exec.shutdown.timeout", "10s", new TimeValidator(TimeUnit.SECONDS), "How long HiveServer2 shutdown will wait for async threads to terminate."), diff --git service/src/java/org/apache/hive/service/cli/session/SessionManager.java service/src/java/org/apache/hive/service/cli/session/SessionManager.java index 5082689..6257c88 100644 --- service/src/java/org/apache/hive/service/cli/session/SessionManager.java +++ service/src/java/org/apache/hive/service/cli/session/SessionManager.java @@ -43,7 +43,6 @@ import org.apache.hadoop.hive.common.metrics.common.MetricsVariable; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.hooks.HookUtils; import org.apache.hadoop.hive.ql.hooks.HooksLoader; import org.apache.hive.service.CompositeService; import org.apache.hive.service.cli.HiveSQLException; @@ -167,7 +166,12 @@ private void initSessionImplClassName() { private void createBackgroundOperationPool() { int poolSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_THREADS); - LOG.info("HiveServer2: Background operation thread pool size: " + poolSize); + int minPoolSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_THREADS_MIN); + if (minPoolSize == -1) { + minPoolSize = poolSize; + } + LOG.info("HiveServer2: Background operation thread pool minimum size: " + minPoolSize); + LOG.info("HiveServer2: Background operation thread pool maximum size: " + poolSize); int poolQueueSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_WAIT_QUEUE_SIZE); LOG.info("HiveServer2: Background operation thread wait queue size: " + poolQueueSize); long keepAliveTime = HiveConf.getTimeVar( @@ -179,8 +183,8 @@ private void createBackgroundOperationPool() { // Threads terminate when they are idle for more than the keepAliveTime // A bounded blocking queue is used to queue incoming operations, if #operations > poolSize String threadPoolName = "HiveServer2-Background-Pool"; - final BlockingQueue queue = new LinkedBlockingQueue(poolQueueSize); - backgroundOperationPool = new ThreadPoolExecutor(poolSize, poolSize, + final BlockingQueue queue = new LinkedBlockingQueue(poolQueueSize); + backgroundOperationPool = new ThreadPoolExecutor(minPoolSize, poolSize, keepAliveTime, TimeUnit.SECONDS, queue, new ThreadFactoryWithGarbageCleanup(threadPoolName)); backgroundOperationPool.allowCoreThreadTimeOut(true); diff --git service/src/test/org/apache/hive/service/cli/TestRetryingThriftCLIServiceClient.java service/src/test/org/apache/hive/service/cli/TestRetryingThriftCLIServiceClient.java index 2c4a6fe..fc0427c 100644 --- service/src/test/org/apache/hive/service/cli/TestRetryingThriftCLIServiceClient.java +++ service/src/test/org/apache/hive/service/cli/TestRetryingThriftCLIServiceClient.java @@ -58,6 +58,7 @@ public void init() { hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE, "binary"); hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_CLIENT_RETRY_LIMIT, 3); hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_CLIENT_CONNECTION_RETRY_LIMIT, 3); + hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_ASYNC_EXEC_THREADS_MIN, 2); hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_ASYNC_EXEC_THREADS, 10); hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_ASYNC_EXEC_SHUTDOWN_TIMEOUT, "1s"); hiveConf diff --git service/src/test/org/apache/hive/service/cli/session/TestSessionManagerMetrics.java service/src/test/org/apache/hive/service/cli/session/TestSessionManagerMetrics.java index efc2e1a..2646f8b 100644 --- service/src/test/org/apache/hive/service/cli/session/TestSessionManagerMetrics.java +++ service/src/test/org/apache/hive/service/cli/session/TestSessionManagerMetrics.java @@ -66,6 +66,7 @@ @Before public void setup() throws Exception { HiveConf conf = new HiveConf(); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_ASYNC_EXEC_THREADS_MIN, 0); conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_ASYNC_EXEC_THREADS, 2); conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_ASYNC_EXEC_WAIT_QUEUE_SIZE, 10); conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_ASYNC_EXEC_KEEPALIVE_TIME, "1000000s");