diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 2723dad..c7150ed 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2581,6 +2581,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "YARN shuffle port for LLAP-daemon-hosted shuffle.", "llap.daemon.yarn.shuffle.port"), LLAP_DAEMON_YARN_CONTAINER_MB("hive.llap.daemon.yarn.container.mb", -1, "llap server yarn container size in MB. Used in LlapServiceDriver and package.py", "llap.daemon.yarn.container.mb"), + LLAP_DAEMON_QUEUE_NAME("hive.llap.daemon.queue.name", null, + "Queue name within which the llap slider application will run. Used in LlapServiceDriver and package.py"), LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED("hive.llap.daemon.shuffle.dir.watcher.enabled", false, "TODO doc", "llap.daemon.shuffle.dir-watcher.enabled"), LLAP_DAEMON_AM_LIVENESS_HEARTBEAT_INTERVAL_MS( diff --git llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java index 1732df8..c43bf97 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java @@ -18,9 +18,7 @@ package org.apache.hadoop.hive.llap.cli; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.util.Properties; import javax.annotation.Nonnull; @@ -31,6 +29,7 @@ import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.hadoop.hive.conf.HiveConf; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.util.StringUtils; @@ -51,6 +50,8 @@ public static final String OPTION_AUXHBASE = "auxhbase"; // used to localize jars public static final String OPTION_JAVA_HOME = "javaHome"; // forward via config.json public static final String OPTION_HIVECONF = "hiveconf"; // llap-daemon-site if relevant parameter + public static final String OPTION_SLIDER_AM_CONTAINER_MB = "slider-am-container-mb"; // forward as arg + public static final String OPTION_LLAP_QUEUE = "queue"; // forward via config.json public class LlapOptions { private final int instances; @@ -64,10 +65,11 @@ private final boolean isHbase; private final Properties conf; private final String javaPath; + private final String llapQueueName; public LlapOptions(String name, int instances, String directory, int executors, long cache, long size, long xmx, String jars, boolean isHbase, - @Nonnull Properties hiveconf, String javaPath) + @Nonnull Properties hiveconf, String javaPath, String llapQueueName) throws ParseException { if (instances <= 0) { throw new ParseException("Invalid configuration: " + instances @@ -84,6 +86,7 @@ public LlapOptions(String name, int instances, String directory, int executors, this.isHbase = isHbase; this.conf = hiveconf; this.javaPath = javaPath; + this.llapQueueName = llapQueueName; } public String getName() { @@ -129,6 +132,10 @@ public Properties getConfig() { public String getJavaPath() { return javaPath; } + + public String getLlapQueueName() { + return llapQueueName; + } } protected static final Logger l4j = LoggerFactory.getLogger(LlapOptionsProcessor.class.getName()); @@ -169,6 +176,10 @@ public LlapOptionsProcessor() { options.addOption(OptionBuilder.hasArg().withArgName(OPTION_XMX).withLongOpt(OPTION_XMX) .withDescription("working memory size").create('w')); + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_LLAP_QUEUE) + .withLongOpt(OPTION_LLAP_QUEUE) + .withDescription("The queue within which LLAP will be started").create('q')); + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_AUXJARS).withLongOpt(OPTION_AUXJARS) .withDescription("additional jars to package (by default, JSON SerDe jar is packaged" + " if available)").create('j')); @@ -187,6 +198,10 @@ public LlapOptionsProcessor() { .withDescription("Use value for given property. Overridden by explicit parameters") .create()); + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SLIDER_AM_CONTAINER_MB) + .withLongOpt(OPTION_SLIDER_AM_CONTAINER_MB) + .withDescription("The size of the slider AppMaster container in MB").create()); + // [-H|--help] options.addOption(new Option("H", "help", false, "Print help information")); } @@ -215,6 +230,9 @@ public LlapOptions processOptions(String argv[]) throws ParseException, IOExcept final long xmx = parseSuffixed(commandLine.getOptionValue(OPTION_XMX, "-1")); final boolean isHbase = Boolean.parseBoolean(commandLine.getOptionValue(OPTION_AUXHBASE, "true")); + final String queueName = commandLine.getOptionValue(OPTION_LLAP_QUEUE, + HiveConf.ConfVars.LLAP_DAEMON_QUEUE_NAME.getDefaultValue()); + final Properties hiveconf; if (commandLine.hasOption(OPTION_HIVECONF)) { @@ -227,10 +245,12 @@ public LlapOptions processOptions(String argv[]) throws ParseException, IOExcept if (commandLine.hasOption(OPTION_JAVA_HOME)) { javaHome = commandLine.getOptionValue(OPTION_JAVA_HOME); } + // loglevel, chaosmonkey & args are parsed by the python processor return new LlapOptions( - name, instances, directory, executors, cache, size, xmx, jars, isHbase, hiveconf, javaHome); + name, instances, directory, executors, cache, size, xmx, jars, isHbase, hiveconf, javaHome, + queueName); } private void printUsage() { diff --git llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java index 46edc42..b2c743f 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java @@ -224,6 +224,12 @@ private void run(String[] args) throws Exception { propsDirectOptions.setProperty(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname, String.valueOf(xmx)); } + if (options.getLlapQueueName() != null && !options.getLlapQueueName().isEmpty()) { + conf.set(ConfVars.LLAP_DAEMON_QUEUE_NAME.varname, options.getLlapQueueName()); + propsDirectOptions + .setProperty(ConfVars.LLAP_DAEMON_QUEUE_NAME.varname, options.getLlapQueueName()); + } + URL logger = conf.getResource(LlapDaemon.LOG4j2_PROPERTIES_FILE); @@ -379,6 +385,12 @@ private void run(String[] args) throws Exception { configs.put(ConfVars.LLAP_DAEMON_NUM_EXECUTORS.varname, HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_NUM_EXECUTORS)); + // Let YARN pick the queue name, if it isn't provided in hive-site, or via the command-line + if (HiveConf.getVar(conf, ConfVars.LLAP_DAEMON_QUEUE_NAME) != null) { + configs.put(ConfVars.LLAP_DAEMON_QUEUE_NAME.varname, + HiveConf.getVar(conf, ConfVars.LLAP_DAEMON_QUEUE_NAME)); + } + configs.put(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, -1)); diff --git llap-server/src/main/resources/package.py llap-server/src/main/resources/package.py index 0204b7b..58c43be 100644 --- llap-server/src/main/resources/package.py +++ llap-server/src/main/resources/package.py @@ -37,6 +37,12 @@ def __init__(self, config): self.container_cores = self.cores self.heap_size = h + if (not config.get("hive.llap.daemon.queue.name","")): + self.queueString = "" + else: + self.queueString = "--queue " + self.queueString += config["hive.llap.daemon.queue.name"] + def __repr__(self): return "" % (self.heap_size, self.container_size) @@ -61,6 +67,7 @@ def main(args): parser.add_argument("--name", default="llap0") parser.add_argument("--loglevel", default="INFO") parser.add_argument("--chaosmonkey", type=int, default=0) + parser.add_argument("--slider-am-container-mb", type=int, default=1024) parser.add_argument("--slider-keytab-dir", default="") parser.add_argument("--slider-keytab", default="") parser.add_argument("--slider-principal", default="") @@ -71,6 +78,7 @@ def main(args): (args, unknown_args) = parser.parse_known_args(args) input = args.input output = args.output + slider_am_jvm_heapsize = max(args.slider_am_container_mb * 0.8, args.slider_am_container_mb - 1024) slider_keytab_dir = args.slider_keytab_dir slider_keytab = args.slider_keytab slider_principal = args.slider_principal @@ -103,9 +111,12 @@ def main(args): "name" : args.name, "daemon_args" : args.args, "daemon_loglevel" : args.loglevel, + "queue.string" : resource.queueString, "monkey_interval" : args.chaosmonkey, "monkey_percentage" : monkey_percentage, "monkey_enabled" : args.chaosmonkey > 0, + "slider.am.container.mb" : args.slider_am_container_mb, + "slider_am_jvm_heapsize" : slider_am_jvm_heapsize, "slider_keytab_dir" : slider_keytab_dir, "slider_keytab" : slider_keytab, "slider_principal" : slider_principal diff --git llap-server/src/main/resources/templates.py llap-server/src/main/resources/templates.py index 277a49c..8baa927 100644 --- llap-server/src/main/resources/templates.py +++ llap-server/src/main/resources/templates.py @@ -88,7 +88,7 @@ }, "components": { "slider-appmaster": { - "jvm.heapsize": "1024M", + "jvm.heapsize": "%(slider_am_jvm_heapsize)dM", "slider.hdfs.keytab.dir": "%(slider_keytab_dir)s", "slider.am.login.keytab.name": "%(slider_keytab)s", "slider.keytab.principal.name": "%(slider_principal)s" @@ -106,6 +106,8 @@ }, "components": { "slider-appmaster": { + "yarn.memory": "%(slider.am.container.mb)d", + "yarn.component.instances": "1" }, "LLAP": { "yarn.role.priority": "1", @@ -126,5 +128,5 @@ slider stop %(name)s slider destroy %(name)s --force || slider destroy %(name)s slider install-package --name LLAP --package $BASEDIR/llap-%(version)s.zip --replacepkg -slider create %(name)s --resources $BASEDIR/resources.json --template $BASEDIR/appConfig.json +slider create %(name)s --resources $BASEDIR/resources.json --template $BASEDIR/appConfig.json %(queue.string)s """