diff --git llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java index 085c977..0705225 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java @@ -171,7 +171,7 @@ public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = "llap.shuffle.connection-keep-alive.enable"; - public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = false; + public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = true; public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT = "llap.shuffle.connection-keep-alive.timeout"; @@ -194,7 +194,7 @@ public static final String MAX_SHUFFLE_THREADS = "llap.shuffle.max.threads"; // 0 implies Netty default of 2 * number of available processors - public static final int DEFAULT_MAX_SHUFFLE_THREADS = 0; + public static final int DEFAULT_MAX_SHUFFLE_THREADS = Runtime.getRuntime().availableProcessors() * 3; public static final String SHUFFLE_BUFFER_SIZE = "llap.shuffle.transfer.buffer.size"; diff --git llap-server/src/main/resources/package.py llap-server/src/main/resources/package.py index 8a378ef..e83d3b0 100644 --- llap-server/src/main/resources/package.py +++ llap-server/src/main/resources/package.py @@ -20,6 +20,7 @@ def __init__(self, config): # convert to Mb self.cache = config["hive.llap.io.memory.size"] / (1024*1024.0) self.direct = config["hive.llap.io.allocator.direct"] + self.executors = config["hive.llap.daemon.num.executors"] self.min_cores = -1 # compute heap + cache as final Xmx h = self.memory @@ -129,10 +130,13 @@ def main(args): config = json_parse(open(join(input, "config.json")).read()) java_home = config["java.home"] max_direct_memory = config["max_direct_memory"] + + resource = LlapResource(config) + daemon_args = args.args if long(max_direct_memory) > 0: daemon_args = " -XX:MaxDirectMemorySize=%s %s" % (max_direct_memory, daemon_args) - resource = LlapResource(config) + daemon_args = " -Dhttp.maxConnections=%s %s" % ((max(args.instances, resource.executors) + 1), daemon_args) # 5% container failure every monkey_interval seconds monkey_percentage = 5 # 5% vars = {