diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapServiceCommandLine.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapServiceCommandLine.java index 5323102282..9d9c594d5a 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapServiceCommandLine.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapServiceCommandLine.java @@ -161,6 +161,25 @@ .hasArg() .create("auxhive"); + private static final Option SKIP_VALIDATE_CONF = OptionBuilder + .withLongOpt("skipValidateConf") + .withDescription("Should LLAP conf setting validation be done") + .hasArg(false) + .create("skipValidateConf"); + + private static final Option PARTIAL_DOWNLOAD = OptionBuilder + .withLongOpt("partialDownload") + .withDescription("Use downloadType option to specify which components to download") + .hasArg(false) + .create("partialDownload"); + + private static final Option DOWNLOADTYPE = OptionBuilder + .withLongOpt("downloadType") + .withDescription("What to download (tezjars|localjars|auxjars|udffile|configs)") + .withArgName("downloadType") + .hasArg() + .create("downloadType"); + private static final Option HELP = OptionBuilder .withLongOpt("help") .withDescription("Print help information") @@ -201,6 +220,9 @@ OPTIONS.addOption(START); OPTIONS.addOption(OUTPUT); OPTIONS.addOption(AUXHIVE); + OPTIONS.addOption(SKIP_VALIDATE_CONF); + OPTIONS.addOption(PARTIAL_DOWNLOAD); + OPTIONS.addOption(DOWNLOADTYPE); OPTIONS.addOption(HELP); OPTIONS.addOption(OptionBuilder @@ -319,6 +341,9 @@ private boolean isStarting; private String output; private boolean isHiveAux; + private boolean skipValidateConf; + private boolean isPartialDownload; + private String[] downloadTypes; private boolean isHelp; static LlapServiceCommandLine parseArguments(String[] args) { @@ -385,6 +410,9 @@ private void parseCommandLine(String[] args) throws ParseException { isStarting = cl.hasOption(START.getOpt()); output = cl.getOptionValue(OUTPUT.getLongOpt()); isHiveAux = Boolean.parseBoolean(cl.getOptionValue(AUXHIVE.getOpt(), "true")); + skipValidateConf = cl.hasOption(SKIP_VALIDATE_CONF.getOpt()); + isPartialDownload = cl.hasOption(PARTIAL_DOWNLOAD.getOpt()); + downloadTypes = cl.getOptionValues(DOWNLOADTYPE.getOpt()); } private static void printUsage() { @@ -467,4 +495,8 @@ boolean isStarting() { String getOutput() { return output; } + + boolean isSkipValidateConf() { return skipValidateConf; } + boolean isPartialDownload() { return isPartialDownload; } + String[] getDownloadTypes() { return downloadTypes; } } diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapServiceDriver.java index bbc7265c40..2990be6fc9 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapServiceDriver.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapServiceDriver.java @@ -117,7 +117,15 @@ private int run() throws Exception { LlapTarComponentGatherer tarComponentGatherer = new LlapTarComponentGatherer(cl, conf, propsDirectOptions, fs, rawFs, executor, tmpDir); tarComponentGatherer.createDirs(); - tarComponentGatherer.submitTarComponentGatherTasks(); + if (cl.isPartialDownload()) { + for (String downloadType : cl.getDownloadTypes()) { + LlapTarComponentGatherer.TaskType taskType = LlapTarComponentGatherer.TaskType.valueOf(downloadType.toUpperCase()); + LOG.info("Adding gather task type {}", taskType); + tarComponentGatherer.submitGatherTask(taskType); + } + } else { + tarComponentGatherer.submitTarComponentGatherTasks(); + } // TODO: need to move from Python to Java for the rest of the script. LlapConfigJsonCreator lcjCreator = new LlapConfigJsonCreator(conf, rawFs, tmpDir, cl.getCache(), cl.getXmx(), @@ -152,7 +160,9 @@ private void setupConf(Properties propsDirectOptions) throws Exception { for (String f : LlapDaemonConfiguration.DAEMON_CONFIGS) { conf.addResource(f); if (conf.getResource(f) == null) { - throw new Exception("Unable to find required config file: " + f); + if (!cl.isSkipValidateConf()) { + throw new Exception("Unable to find required config file: " + f); + } } } for (String f : LlapDaemonConfiguration.SSL_DAEMON_CONFIGS) { @@ -181,25 +191,27 @@ private void setupConf(Properties propsDirectOptions) throws Exception { String sizeStr = LlapUtil.humanReadableByteCount(cl.getSize()); String xmxStr = LlapUtil.humanReadableByteCount(cl.getXmx()); - if (cl.getSize() != -1) { - if (cl.getCache() != -1) { - if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.LLAP_ALLOCATOR_MAPPED)) { - // direct heap allocations need to be safer - Preconditions.checkArgument(cl.getCache() < cl.getSize(), "Cache size (" + cacheStr + ") has to be smaller" + - " than the container sizing (" + sizeStr + ")"); - } else if (cl.getCache() < cl.getSize()) { - LOG.warn("Note that this might need YARN physical memory monitoring to be turned off " - + "(yarn.nodemanager.pmem-check-enabled=false)"); + if (!cl.isSkipValidateConf()) { + if (cl.getSize() != -1) { + if (cl.getCache() != -1) { + if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.LLAP_ALLOCATOR_MAPPED)) { + // direct heap allocations need to be safer + Preconditions.checkArgument(cl.getCache() < cl.getSize(), "Cache size (" + cacheStr + ") has to be smaller" + + " than the container sizing (" + sizeStr + ")"); + } else if (cl.getCache() < cl.getSize()) { + LOG.warn("Note that this might need YARN physical memory monitoring to be turned off " + + "(yarn.nodemanager.pmem-check-enabled=false)"); + } + } + if (cl.getXmx() != -1) { + Preconditions.checkArgument(cl.getXmx() < cl.getSize(), "Working memory (Xmx=" + xmxStr + ") has to be" + + " smaller than the container sizing (" + sizeStr + ")"); + } + if (isDirect && !HiveConf.getBoolVar(conf, HiveConf.ConfVars.LLAP_ALLOCATOR_MAPPED)) { + // direct and not memory mapped + Preconditions.checkArgument(cl.getXmx() + cl.getCache() <= cl.getSize(), "Working memory (Xmx=" + + xmxStr + ") + cache size (" + cacheStr + ") has to be smaller than the container sizing (" + sizeStr + ")"); } - } - if (cl.getXmx() != -1) { - Preconditions.checkArgument(cl.getXmx() < cl.getSize(), "Working memory (Xmx=" + xmxStr + ") has to be" + - " smaller than the container sizing (" + sizeStr + ")"); - } - if (isDirect && !HiveConf.getBoolVar(conf, HiveConf.ConfVars.LLAP_ALLOCATOR_MAPPED)) { - // direct and not memory mapped - Preconditions.checkArgument(cl.getXmx() + cl.getCache() <= cl.getSize(), "Working memory (Xmx=" + - xmxStr + ") + cache size (" + cacheStr + ") has to be smaller than the container sizing (" + sizeStr + ")"); } } @@ -244,8 +256,10 @@ private void setupConf(Properties propsDirectOptions) throws Exception { long containerSizeMB = containerSize / (1024 * 1024); long minAllocMB = conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, -1); String containerSizeStr = LlapUtil.humanReadableByteCount(containerSize); - Preconditions.checkArgument(containerSizeMB >= minAllocMB, "Container size (" + containerSizeStr + ") should be " + - "greater than minimum allocation(" + LlapUtil.humanReadableByteCount(minAllocMB * 1024L * 1024L) + ")"); + if (!cl.isSkipValidateConf()) { + Preconditions.checkArgument(containerSizeMB >= minAllocMB, "Container size (" + containerSizeStr + ") should be " + + "greater than minimum allocation(" + LlapUtil.humanReadableByteCount(minAllocMB * 1024L * 1024L) + ")"); + } conf.setLong(ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB.varname, containerSizeMB); propsDirectOptions.setProperty(ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB.varname, String.valueOf(containerSizeMB)); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapTarComponentGatherer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapTarComponentGatherer.java index a83647bece..fd2ea3471b 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapTarComponentGatherer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapTarComponentGatherer.java @@ -35,7 +35,7 @@ /** * Gathers all the jar files necessary to start llap. */ -class LlapTarComponentGatherer { +public class LlapTarComponentGatherer { private static final Logger LOG = LoggerFactory.getLogger(LlapTarComponentGatherer.class.getName()); //using Callable instead of Runnable to be able to throw Exception @@ -52,6 +52,10 @@ private final Path udfDir; private final Path confDir; + enum TaskType { + TEZJARS, LOCALJARS, AUXJARS, UDFFILE, CONFIGS, + } + LlapTarComponentGatherer(LlapServiceCommandLine cl, HiveConf conf, Properties directProperties, FileSystem fs, FileSystem rawFs, ExecutorService executor, Path tmpDir) { this.cl = cl; @@ -78,6 +82,30 @@ void createDirs() throws Exception { } } + void submitGatherTask(TaskType taskType) throws Exception { + CompletionService asyncRunner = new ExecutorCompletionService(executor); + switch (taskType) { + case TEZJARS: + tasks.put("downloadTezJars", asyncRunner.submit(new AsyncTaskDownloadTezJars(conf, fs, rawFs, libDir, tezDir))); + break; + case LOCALJARS: + tasks.put("copyLocalJars", asyncRunner.submit(new AsyncTaskCopyLocalJars(rawFs, libDir))); + break; + case AUXJARS: + tasks.put("copyAuxJars", asyncRunner.submit(new AsyncTaskCopyAuxJars(cl, conf, rawFs, libDir))); + break; + case UDFFILE: + tasks.put("createUdfFile", asyncRunner.submit(new AsyncTaskCreateUdfFile(conf, fs, rawFs, udfDir, confDir))); + break; + case CONFIGS: + tasks.put("copyConfigs", asyncRunner.submit(new AsyncTaskCopyConfigs(cl, conf, directProperties, rawFs, + confDir))); + break; + default: + throw new IllegalArgumentException("Unexpected download tasktype " + taskType); + } + } + void submitTarComponentGatherTasks() { CompletionService asyncRunner = new ExecutorCompletionService(executor);