diff --git llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java index 8cd6df7..fce1275 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java @@ -98,6 +98,7 @@ public LlapServiceDriver() { } public static void main(String[] args) throws Exception { + LOG.info("llap service driver invoked with arguments={}", args); int ret = 0; try { new LlapServiceDriver().run(args); @@ -105,6 +106,8 @@ public static void main(String[] args) throws Exception { System.err.println("Failed: " + t.getMessage()); t.printStackTrace(); ret = 3; + } finally { + LOG.info("llap service driver finished"); } if (LOG.isDebugEnabled()) { LOG.debug("Completed processing - exiting with " + ret); @@ -134,7 +137,7 @@ private static void populateConf(Configuration configured, Configuration target, } } - private static void populateConfWithLlapProperties(Configuration conf, Properties properties) { + static void populateConfWithLlapProperties(Configuration conf, Properties properties) { for(Entry props : properties.entrySet()) { String key = (String) props.getKey(); if (HiveConf.getLlapDaemonConfVars().contains(key)) { diff --git llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java index e3a100c..5e96a73 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.llap.cli; +import java.util.Properties; + import jline.TerminalFactory; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; @@ -83,14 +85,20 @@ public int getNumArgs() { public static class LlapStatusOptions { private final String name; + private final Properties conf; - LlapStatusOptions(String name) { + LlapStatusOptions(String name, Properties hiveProperties) { this.name = name; + this.conf = hiveProperties; } public String getName() { return name; } + + public Properties getConf() { + return conf; + } } private final Options options = new Options(); @@ -113,14 +121,20 @@ public LlapStatusOptionsProcessor() { public LlapStatusOptions processOptions(String[] args) throws ParseException { commandLine = new GnuParser().parse(options, args); - if (commandLine.hasOption(OptionConstants.HELP.getShortOpt()) || - false == commandLine.hasOption(OptionConstants.NAME.getLongOpt())) { + if (commandLine.hasOption(OptionConstants.HELP.getShortOpt())) { printUsage(); return null; } String name = commandLine.getOptionValue(OptionConstants.NAME.getLongOpt()); - return new LlapStatusOptions(name); + Properties hiveConf; + if (commandLine.hasOption(OptionConstants.HIVECONF.getLongOpt())) { + hiveConf = commandLine.getOptionProperties(OptionConstants.HIVECONF.getLongOpt()); + } else { + hiveConf = new Properties(); + } + + return new LlapStatusOptions(name, hiveConf); } diff --git llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java index 45ba5d0..bb8137b 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java @@ -26,7 +26,10 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Properties; +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.llap.cli.LlapStatusOptionsProcessor.LlapStatusOptions; @@ -60,7 +63,8 @@ private final Configuration conf; private final Clock clock = new SystemClock(); - private final AppStatusBuilder appStatusBuilder = new AppStatusBuilder(); + @VisibleForTesting + final AppStatusBuilder appStatusBuilder = new AppStatusBuilder(); public LlapStatusServiceDriver() { SessionState ss = SessionState.get(); @@ -85,7 +89,23 @@ public int run(String[] args) { conf.addResource(f); } conf.reloadConfiguration(); + for (Map.Entry props : options.getConf().entrySet()) { + conf.set((String) props.getKey(), (String) props.getValue()); + } + String appName; + appName = options.getName(); + if (StringUtils.isEmpty(appName)) { + appName = HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS); + if (appName.startsWith("@") && appName.length() > 1) { + appName = appName.substring(1); + } + } + if (StringUtils.isEmpty(appName) || (appName.startsWith("@") || appName.length() == 1)) { + String message = "Invalid app name. This must be setup via config or passed in as a parameter"; + LOG.info(message); + return ExitCode.INCORRECT_USAGE.getInt(); + } try { sliderClient = createSliderClient(); @@ -97,7 +117,7 @@ public int run(String[] args) { // Get the App report from YARN ApplicationReport appReport = null; try { - appReport = getAppReport(options, sliderClient, FIND_YARN_APP_TIMEOUT); + appReport = getAppReport(appName, sliderClient, FIND_YARN_APP_TIMEOUT); } catch (LlapStatusCliException e) { logError(e); return e.getExitCode().getInt(); @@ -120,7 +140,7 @@ public int run(String[] args) { } else { // Get information from slider. try { - ret = populateAppStatusFromSlider(options, sliderClient, appStatusBuilder); + ret = populateAppStatusFromSlider(appName, sliderClient, appStatusBuilder); } catch (LlapStatusCliException e) { // In case of failure, send back whatever is constructed sop far - which wouldbe from the AppReport logError(e); @@ -140,8 +160,8 @@ public int run(String[] args) { } return ret.getInt(); }finally { - if (LOG.isTraceEnabled()) { - LOG.trace("Final AppState: " + appStatusBuilder.toString()); + if (LOG.isDebugEnabled()) { + LOG.debug("Final AppState: " + appStatusBuilder.toString()); } if (sliderClient != null) { sliderClient.stop(); @@ -157,6 +177,7 @@ public void outputJson(PrintWriter writer) throws LlapStatusCliException { try { writer.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(appStatusBuilder)); } catch (IOException e) { + LOG.warn("Failed to create JSON", e); throw new LlapStatusCliException(ExitCode.LLAP_JSON_GENERATION_ERROR, "Failed to create JSON", e); } @@ -185,7 +206,7 @@ public void serviceInit(Configuration conf) throws Exception { } - private ApplicationReport getAppReport(LlapStatusOptions options, SliderClient sliderClient, + private ApplicationReport getAppReport(String appName, SliderClient sliderClient, long timeoutMs) throws LlapStatusCliException { long startTime = clock.getTime(); @@ -196,7 +217,7 @@ private ApplicationReport getAppReport(LlapStatusOptions options, SliderClient s // move to running state. Potentially even wait for the containers to be launched. while (clock.getTime() < timeoutTime && appReport == null) { try { - appReport = sliderClient.getYarnAppListClient().findInstance(options.getName()); + appReport = sliderClient.getYarnAppListClient().findInstance(appName); if (appReport == null) { long remainingTime = Math.min(timeoutTime - clock.getTime(), 500l); if (remainingTime > 0) { @@ -263,18 +284,18 @@ private ExitCode processAppReport(ApplicationReport appReport, /** * - * @param options + * @param appName * @param sliderClient * @param appStatusBuilder * @return an ExitCode. An ExitCode other than ExitCode.SUCCESS implies future progress not possible * @throws LlapStatusCliException */ - private ExitCode populateAppStatusFromSlider(LlapStatusOptions options, SliderClient sliderClient, AppStatusBuilder appStatusBuilder) throws + private ExitCode populateAppStatusFromSlider(String appName, SliderClient sliderClient, AppStatusBuilder appStatusBuilder) throws LlapStatusCliException { ClusterDescription clusterDescription; try { - clusterDescription = sliderClient.getClusterDescription(options.getName()); + clusterDescription = sliderClient.getClusterDescription(appName); } catch (SliderException e) { throw new LlapStatusCliException(ExitCode.SLIDER_CLIENT_ERROR_OTHER, "Failed to get cluster description from slider. SliderErrorCode=" + (e).getExitCode(), e); @@ -801,17 +822,27 @@ private static void logError(Throwable t) { public static void main(String[] args) { + LOG.info("llap status invoked with arguments = {}", args); int ret; try { LlapStatusServiceDriver statusServiceDriver = new LlapStatusServiceDriver(); ret = statusServiceDriver.run(args); if (ret == ExitCode.SUCCESS.getInt()) { - statusServiceDriver.outputJson(new PrintWriter(System.out)); + try (PrintWriter pw = new PrintWriter(System.out)) { + statusServiceDriver.outputJson(pw); + } } } catch (Throwable t) { logError(t); - ret = ExitCode.INTERNAL_ERROR.getInt(); + if (t instanceof LlapStatusCliException) { + LlapStatusCliException ce = (LlapStatusCliException) t; + ret = ce.getExitCode().getInt(); + } else { + ret = ExitCode.INTERNAL_ERROR.getInt(); + } + } finally { + LOG.info("llap status finished"); } if (LOG.isDebugEnabled()) { LOG.debug("Completed processing - exiting with " + ret); diff --git llap-server/src/main/resources/llap-cli-log4j2.properties llap-server/src/main/resources/llap-cli-log4j2.properties index a141042..2f27b5e 100644 --- llap-server/src/main/resources/llap-cli-log4j2.properties +++ llap-server/src/main/resources/llap-cli-log4j2.properties @@ -22,7 +22,7 @@ packages = org.apache.hadoop.hive.ql.log property.hive.log.level = INFO property.hive.root.logger = console property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} -property.hive.log.file = hive.log +property.hive.log.file = llap-cli.log # list of all appenders appenders = console, DRFA diff --git packaging/src/main/assembly/bin.xml packaging/src/main/assembly/bin.xml index 97bef59..8fd934a 100644 --- packaging/src/main/assembly/bin.xml +++ packaging/src/main/assembly/bin.xml @@ -393,6 +393,11 @@ llap-daemon-log4j2.properties.template + ${project.parent.basedir}/llap-server/src/main/resources/llap-cli-log4j2.properties + conf + llap-cli-log4j2.properties.template + + ${project.parent.basedir}/hcatalog/README.txt hcatalog/share/doc/hcatalog