diff --git bin/ext/llap.sh bin/ext/llap.sh index 93cd753..fa87587 100644 --- bin/ext/llap.sh +++ bin/ext/llap.sh @@ -37,10 +37,13 @@ llap () { test -f $TMPDIR/config.json python $HIVE_HOME/scripts/llap/slider/package.py --input $TMPDIR "$@" + + # remove temp files + rm -rf $TMPDIR } llap_help () { - CLASS=org.apache.hadoop.hive.llap.ServiceDriver + CLASS=org.apache.hadoop.hive.llap.cli.LlapServiceDriver; execHiveCmd $CLASS "--help" } diff --git llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java index 825771d..5e3a3b5 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java @@ -35,8 +35,10 @@ public class LlapOptions { private int instances = 0; private String directory = null; + private String name; + private String args; - public LlapOptions(int instances, String directory) + public LlapOptions(String name, int instances, String directory) throws ParseException { if (instances <= 0) { throw new ParseException("Invalid configuration: " + instances @@ -44,9 +46,14 @@ public LlapOptions(int instances, String directory) } this.instances = instances; this.directory = directory; + this.name = name; } - public int getNumInstances() { + public String getName() { + return name; + } + + public int getInstances() { return instances; } @@ -67,11 +74,17 @@ public LlapOptionsProcessor() { options.addOption(OptionBuilder.hasArg().withArgName("instances").withLongOpt("instances") .withDescription("Specify the number of instances to run this on").create('i')); - // [-H|--help] - options.addOption(new Option("H", "help", false, "Print help information")); + options.addOption(OptionBuilder.hasArg().withArgName("name").withLongOpt("name") + .withDescription("Cluster name for YARN registry").create('n')); options.addOption(OptionBuilder.hasArg().withArgName("directory").withLongOpt("directory") .withDescription("Temp directory for jars etc.").create('d')); + + options.addOption(OptionBuilder.hasArg().withArgName("args").withLongOpt("args") + .withDescription("java arguments to the llap instance").create('a')); + + // [-H|--help] + options.addOption(new Option("H", "help", false, "Print help information")); } public LlapOptions processOptions(String argv[]) throws ParseException { @@ -84,7 +97,9 @@ public LlapOptions processOptions(String argv[]) throws ParseException { int instances = Integer.parseInt(commandLine.getOptionValue("instances")); String directory = commandLine.getOptionValue("directory"); - return new LlapOptions(instances, directory); + String name = commandLine.getOptionValue("name", null); + + return new LlapOptions(name, instances, directory); } private void printUsage() { diff --git llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java index 2f7259c..ca22a21 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java @@ -60,9 +60,40 @@ public static void main(String[] args) throws Exception { System.exit(ret); } + /** + * Intersect llap-daemon-site.xml configuration properties against an existing Configuration + * object, while resolving any ${} parameters that might be present. + * + * @param raw + * @return configuration object which is a slice of configured + */ + public static Configuration resolve(Configuration configured, String first, String... resources) { + Configuration defaults = new Configuration(false); + + defaults.addResource(first); + + for (String resource : resources) { + defaults.addResource(resource); + } + + Configuration slice = new Configuration(false); + // for everything in defaults, slice out those from the configured + for (Map.Entry kv : defaults) { + slice.set(kv.getKey(), configured.getRaw(kv.getKey())); + } + + return slice; + } + private int run(String[] args) throws Exception { LlapOptionsProcessor optionsProcessor = new LlapOptionsProcessor(); LlapOptions options = optionsProcessor.processOptions(args); + + if (options == null) { + // help + return 0; + } + Path tmpDir = new Path(options.getDirectory()); if (conf == null) { @@ -87,6 +118,12 @@ private int run(String[] args) throws Exception { conf.reloadConfiguration(); + if (options.getName() != null) { + // update service registry configs - caveat: this has nothing to do with the actual settings as read by the AM + // if needed, use --hiveconf llap.daemon.service.hosts=@llap0 to dynamically switch between instances + conf.set(LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_HOSTS, "@" + options.getName()); + } + URL logger = conf.getResource("llap-daemon-log4j.properties"); if (null == logger) { @@ -115,8 +152,7 @@ private int run(String[] args) throws Exception { } lfs.mkdirs(libDir); fs.copyToLocalFile(new Path(tezLibs), new Path(libDir, "tez.tar.gz")); - CompressionUtils.unTar(new Path(libDir, "tez.tar.gz").toString(), - libDir.toString(), true); + CompressionUtils.unTar(new Path(libDir, "tez.tar.gz").toString(), libDir.toString(), true); lfs.delete(new Path(libDir, "tez.tar.gz"), false); // TODO: aux jars (like compression libs) @@ -128,8 +164,17 @@ private int run(String[] args) throws Exception { lfs.mkdirs(confPath); for (String f : neededConfig) { - // they will be file:// URLs - lfs.copyFromLocalFile(new Path(conf.getResource(f).toString()), confPath); + if (f.equals("llap-daemon-site.xml")) { + FSDataOutputStream confStream = lfs.create(new Path(confPath, f)); + + Configuration copy = resolve(conf, "llap-daemon-site.xml"); + + copy.writeXml(confStream); + confStream.close(); + } else { + // they will be file:// URLs + lfs.copyFromLocalFile(new Path(conf.getResource(f).toString()), confPath); + } } lfs.copyFromLocalFile(new Path(logger.toString()), confPath); diff --git llap-server/src/main/resources/llap.py llap-server/src/main/resources/llap.py index fa3bb48..601dfb3 100644 --- llap-server/src/main/resources/llap.py +++ llap-server/src/main/resources/llap.py @@ -48,6 +48,7 @@ def start(self, env): os.environ['LLAP_DAEMON_HEAPSIZE'] = format("{memory_val}") os.environ['LLAP_DAEMON_PID_DIR'] = dirname(format("{pid_file}")) os.environ['LLAP_DAEMON_LD_PATH'] = format('{library_path}') + os.environ['LLAP_DAEMON_OPTS'] = format('{daemon_args}') print "Debug from LLAP python script" print os.environ['LLAP_DAEMON_CONF_DIR'] self.configure(env) diff --git llap-server/src/main/resources/package.py llap-server/src/main/resources/package.py index 7424440..6790464 100644 --- llap-server/src/main/resources/package.py +++ llap-server/src/main/resources/package.py @@ -47,21 +47,27 @@ def zipdir(path, zip, prefix="."): zip.write(src, dst) def main(args): - opts, args = getopt(args,"n:o:i:",["instances=","output=", "input="]) + opts, args = getopt(args,"",["instances=","output=", "input=","args=","name="]) version = os.getenv("HIVE_VERSION") if not version: version = strftime("%d%b%Y", gmtime()) home = os.getenv("HIVE_HOME") output = "llap-slider-%(version)s" % ({"version": version}) instances=1 + name = "llap0" + d_args = "" input = None for k,v in opts: - if k in ("--input", "-i"): + if k in ("--input"): input = v - elif k in ("--output", "-o"): + elif k in ("--output"): output = v - elif k in ("--instances", "-n"): + elif k in ("--instances"): instances = int(v) + elif k in ("--name"): + name = v + elif k in ("--args"): + d_args = v if not input: print "Cannot find input files" sys.exit(1) @@ -77,7 +83,8 @@ def main(args): "container.cores" : resource.container_cores, "hadoop_home" : os.getenv("HADOOP_HOME"), "java_home" : os.getenv("JAVA_HOME"), - "name" : "llap0" + "name" : name, + "daemon_args" : d_args } if not exists(output): diff --git llap-server/src/main/resources/params.py llap-server/src/main/resources/params.py index 8aa8e3c..802baf3 100644 --- llap-server/src/main/resources/params.py +++ llap-server/src/main/resources/params.py @@ -30,7 +30,9 @@ additional_cp = config['configurations']['global']['additional_cp'] app_log_dir = config['configurations']['global']['app_log_dir'] app_log_level = config['configurations']['global']['app_log_level'] +daemon_args = config['configurations']['global']['daemon_args'] port = config['configurations']['global']['listen_port'] memory_val = config['configurations']['global']['memory_val'] library_path = config['configurations']['global']['library_path'] + diff --git llap-server/src/main/resources/templates.py llap-server/src/main/resources/templates.py index e3599c0..7fcb95f 100644 --- llap-server/src/main/resources/templates.py +++ llap-server/src/main/resources/templates.py @@ -75,6 +75,7 @@ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/", "site.global.app_log_level": "INFO,RFA", "site.global.additional_cp": "%(hadoop_home)s", + "site.global.daemon_args": "%(daemon_args)s", "site.global.library_path": "%(hadoop_home)s/lib/native", "site.global.memory_val": "%(heap)d", "site.global.pid_file": "${AGENT_WORK_ROOT}/app/run/llap-daemon.pid"