diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index bfd88f8..85c084b 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -25,9 +25,11 @@ import java.io.PrintStream; import java.net.URL; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -39,6 +41,7 @@ import javax.security.auth.login.LoginException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.hive.conf.Validator.PatternSet; import org.apache.hadoop.hive.conf.Validator.RangeValidator; @@ -109,8 +112,17 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { for (ConfVars confVar : ConfVars.values()) { vars.put(confVar.varname, confVar); } + + Set llapDaemonConfVarsSetLocal = new LinkedHashSet<>(); + populateLlapDaemonVarsSet(llapDaemonConfVarsSetLocal); + llapDaemonVarsSet = Collections.unmodifiableSet(llapDaemonConfVarsSetLocal); } + @InterfaceAudience.Private + public static final String PREFIX_LLAP = "llap."; + @InterfaceAudience.Private + public static final String PREFIX_HIVE_LLAP = "hive.llap."; + /** * Metastore related options that the db is initialized against. When a conf * var in this is list is changed, the metastore instance for the CLI will @@ -237,6 +249,70 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { }; /** + * Variables used by LLAP daemons. + * TODO: Eventually auto-populate this based on prefixes. The conf variables + * will need to be renamed for this. + */ + private static final Set llapDaemonVarsSet; + + private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal) { + llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_ENABLED.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_MEMORY_MODE.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_ALLOCATOR_MIN_ALLOC.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_ALLOCATOR_MAX_ALLOC.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_ALLOCATOR_ARENA_COUNT.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_MEMORY_MAX_SIZE.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_ALLOCATOR_DIRECT.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_USE_LRFU.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_LRFU_LAMBDA.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_CACHE_ALLOW_SYNTHETIC_FILEID.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_USE_FILEID_PATH.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_ORC_ENABLE_TIME_COUNTERS.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_THREADPOOL_SIZE.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_KERBEROS_PRINCIPAL.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_KERBEROS_KEYTAB_FILE.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_ZKSM_KERBEROS_PRINCIPAL.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_ZKSM_KERBEROS_KEYTAB_FILE.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_ZKSM_ZK_CONNECTION_STRING.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_SECURITY_ACL.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_MANAGEMENT_ACL.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DELEGATION_TOKEN_LIFETIME.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_MANAGEMENT_RPC_PORT.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_WEB_AUTO_AUTH.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_RPC_NUM_HANDLERS.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_WORK_DIRS.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_YARN_SHUFFLE_PORT.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_AM_LIVENESS_HEARTBEAT_INTERVAL_MS.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_AM_LIVENESS_CONNECTION_TIMEOUT_MS.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_AM_LIVENESS_CONNECTION_SLEEP_BETWEEN_RETRIES_MS.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_NUM_EXECUTORS.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_RPC_PORT.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_VCPUS_PER_INSTANCE.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_NUM_FILE_CLEANER_THREADS.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_FILE_CLEANUP_DELAY_SECONDS.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_SERVICE_HOSTS.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_SERVICE_REFRESH_INTERVAL.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_ALLOW_PERMANENT_FNS.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_WAIT_QUEUE_COMPARATOR_CLASS_NAME.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_TASK_SCHEDULER_ENABLE_PREEMPTION.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_WEB_PORT.name()); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_WEB_SSL.name()); + } + + /** + * Get a set containing configuration parameter names used by LLAP Server isntances + * @return an unmodifiable set containing llap ConfVars + */ + public static final Set getLlapDaemonConfVars() { + return llapDaemonVarsSet; + } + + + /** * ConfVars. * * These are the default configuration properties for Hive. Each HiveConf @@ -2470,7 +2546,7 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { LLAP_DAEMON_YARN_SHUFFLE_PORT("hive.llap.daemon.yarn.shuffle.port", 15551, "YARN shuffle port for LLAP-daemon-hosted shuffle.", "llap.daemon.yarn.shuffle.port"), LLAP_DAEMON_YARN_CONTAINER_MB("hive.llap.daemon.yarn.container.mb", -1, - "TODO doc. Unused?", "llap.daemon.yarn.container.mb"), + "llap server yarn container size in MB. Used in LlapServiceDriver and package.py", "llap.daemon.yarn.container.mb"), LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED("hive.llap.daemon.shuffle.dir.watcher.enabled", false, "TODO doc", "llap.daemon.shuffle.dir-watcher.enabled"), LLAP_DAEMON_AM_LIVENESS_HEARTBEAT_INTERVAL_MS( diff --git llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java deleted file mode 100644 index abdbc09..0000000 --- llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.llap.configuration; - -import java.net.URL; - -import org.apache.hadoop.conf.Configuration; - -public class LlapConfiguration extends Configuration { - public static final String LLAP_PREFIX = "llap."; - public static final String LLAP_DAEMON_PREFIX = "llap.daemon."; - - public LlapConfiguration(Configuration conf) { - super(conf); - addResource(LLAP_DAEMON_SITE); - } - - public LlapConfiguration() { - super(false); - addResource(LLAP_DAEMON_SITE); - } - - public LlapConfiguration(Configuration conf, URL llapDaemonConfLocation) { - super(conf); - addResource(llapDaemonConfLocation); - } - - private static final String LLAP_DAEMON_SITE = "llap-daemon-site.xml"; -} diff --git llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java index 92044bb..c3c16c4 100644 --- llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java +++ llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java @@ -31,7 +31,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.llap.configuration.LlapConfiguration; import org.apache.hadoop.hive.llap.registry.ServiceInstance; import org.apache.hadoop.hive.llap.registry.ServiceInstanceSet; import org.apache.hadoop.hive.llap.registry.ServiceRegistry; @@ -67,9 +66,8 @@ public LlapFixedRegistryImpl(String hosts, Configuration conf) { this.mngPort = HiveConf.getIntVar(conf, ConfVars.LLAP_MANAGEMENT_RPC_PORT); for (Map.Entry kv : conf) { - if (kv.getKey().startsWith(LlapConfiguration.LLAP_DAEMON_PREFIX) - || kv.getKey().startsWith("hive.llap.") - || kv.getKey().startsWith(LlapConfiguration.LLAP_PREFIX)) { + if (kv.getKey().startsWith(HiveConf.PREFIX_LLAP) + || kv.getKey().startsWith(HiveConf.PREFIX_HIVE_LLAP)) { // TODO: read this somewhere useful, like the task scheduler srv.put(kv.getKey(), kv.getValue()); } diff --git llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapYarnRegistryImpl.java llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapYarnRegistryImpl.java index efe31cc..c83dd6e 100644 --- llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapYarnRegistryImpl.java +++ llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapYarnRegistryImpl.java @@ -39,7 +39,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.llap.configuration.LlapConfiguration; import org.apache.hadoop.hive.llap.registry.ServiceInstance; import org.apache.hadoop.hive.llap.registry.ServiceInstanceSet; import org.apache.hadoop.hive.llap.registry.ServiceRegistry; @@ -161,8 +160,8 @@ public void register() throws IOException { srv.addExternalEndpoint(getServicesEndpoint()); for (Map.Entry kv : this.conf) { - if (kv.getKey().startsWith(LlapConfiguration.LLAP_DAEMON_PREFIX) - || kv.getKey().startsWith("hive.llap.")) { + if (kv.getKey().startsWith(HiveConf.PREFIX_LLAP) + || kv.getKey().startsWith(HiveConf.PREFIX_HIVE_LLAP)) { // TODO: read this somewhere useful, like the task scheduler srv.set(kv.getKey(), kv.getValue()); } diff --git llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java index 6d25384..6529169 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java @@ -18,6 +18,10 @@ package org.apache.hadoop.hive.llap.cli; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -36,6 +40,22 @@ public class LlapOptionsProcessor { + public static final String OPTION_INSTANCES = "instances"; //forward as arg + public static final String OPTION_NAME = "name"; // forward as arg + public static final String OPTION_DIRECTORY = "directory"; // work-dir + public static final String OPTION_ARGS = "args"; // forward as arg + public static final String OPTION_LOGLEVEL = "loglevel"; // forward as arg + public static final String OPTION_CHAOS_MONKEY = "chaosmonkey"; // forward as arg + public static final String OPTION_EXECUTORS = "executors"; // llap-daemon-site + public static final String OPTION_CACHE = "cache"; // llap-daemon-site + public static final String OPTION_SIZE = "size"; // forward via config.json + public static final String OPTION_XMX = "xmx"; // forward as arg + public static final String OPTION_AUXJARS = "auxjars"; // used to localize jars + public static final String OPTION_AUXHBASE = "auxhbase"; // used to localize jars + public static final String OPTION_JAVA_HOME = "javaHome"; // forward via config.json + public static final String OPTION_LLAP_PROPERTIES_FILE = "llapPropertiesFile"; // llap-daemon-site + public static final String OPTION_HIVECONF = "hiveconf"; // llap-daemon-site if relevant parameter + public class LlapOptions { private final int instances; private final String directory; @@ -47,10 +67,14 @@ private final String jars; private final boolean isHbase; private final Properties conf; + private final Properties propertiesFromFile; + private final String javaPath; public LlapOptions(String name, int instances, String directory, int executors, long cache, - long size, long xmx, String jars, boolean isHbase, @Nonnull Properties hiveconf) - throws ParseException { + long size, long xmx, String jars, boolean isHbase, + @Nonnull Properties hiveconf, @Nonnull Properties propertiesFromFile, + String javaPath) + throws ParseException { if (instances <= 0) { throw new ParseException("Invalid configuration: " + instances + " (should be greater than 0)"); @@ -65,6 +89,8 @@ public LlapOptions(String name, int instances, String directory, int executors, this.jars = jars; this.isHbase = isHbase; this.conf = hiveconf; + this.propertiesFromFile = propertiesFromFile; + this.javaPath = javaPath; } public String getName() { @@ -106,57 +132,77 @@ public boolean getIsHBase() { public Properties getConfig() { return conf; } + + public Properties getPropertiesFromFile() { + return propertiesFromFile; + } + + public String getJavaPath() { + return javaPath; + } } protected static final Logger l4j = LoggerFactory.getLogger(LlapOptionsProcessor.class.getName()); private final Options options = new Options(); - Map hiveVariables = new HashMap(); private org.apache.commons.cli.CommandLine commandLine; @SuppressWarnings("static-access") public LlapOptionsProcessor() { // set the number of instances on which llap should run - options.addOption(OptionBuilder.hasArg().withArgName("instances").withLongOpt("instances") + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_INSTANCES).withLongOpt(OPTION_INSTANCES) .withDescription("Specify the number of instances to run this on").create('i')); - options.addOption(OptionBuilder.hasArg().withArgName("name").withLongOpt("name") + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_NAME).withLongOpt(OPTION_NAME) .withDescription("Cluster name for YARN registry").create('n')); - options.addOption(OptionBuilder.hasArg().withArgName("directory").withLongOpt("directory") + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_DIRECTORY).withLongOpt(OPTION_DIRECTORY) .withDescription("Temp directory for jars etc.").create('d')); - options.addOption(OptionBuilder.hasArg().withArgName("args").withLongOpt("args") + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_ARGS).withLongOpt(OPTION_ARGS) .withDescription("java arguments to the llap instance").create('a')); - options.addOption(OptionBuilder.hasArg().withArgName("loglevel").withLongOpt("loglevel") + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_LOGLEVEL).withLongOpt(OPTION_LOGLEVEL) .withDescription("log levels for the llap instance").create('l')); - options.addOption(OptionBuilder.hasArg().withArgName("chaosmonkey").withLongOpt("chaosmonkey") + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_CHAOS_MONKEY).withLongOpt(OPTION_CHAOS_MONKEY) .withDescription("chaosmonkey interval").create('m')); - options.addOption(OptionBuilder.hasArg().withArgName("executors").withLongOpt("executors") + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_EXECUTORS).withLongOpt(OPTION_EXECUTORS) .withDescription("executor per instance").create('e')); - options.addOption(OptionBuilder.hasArg().withArgName("cache").withLongOpt("cache") + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_CACHE).withLongOpt(OPTION_CACHE) .withDescription("cache size per instance").create('c')); - options.addOption(OptionBuilder.hasArg().withArgName("size").withLongOpt("size") + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SIZE).withLongOpt(OPTION_SIZE) .withDescription("container size per instance").create('s')); - options.addOption(OptionBuilder.hasArg().withArgName("xmx").withLongOpt("xmx") + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_XMX).withLongOpt(OPTION_XMX) .withDescription("working memory size").create('w')); - options.addOption(OptionBuilder.hasArg().withArgName("auxjars").withLongOpt("auxjars") + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_AUXJARS).withLongOpt(OPTION_AUXJARS) .withDescription("additional jars to package (by default, JSON SerDe jar is packaged" + " if available)").create('j')); - options.addOption(OptionBuilder.hasArg().withArgName("auxhbase").withLongOpt("auxhbase") + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_AUXHBASE).withLongOpt(OPTION_AUXHBASE) .withDescription("whether to package the HBase jars (true by default)").create('h')); + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_JAVA_HOME).withLongOpt(OPTION_JAVA_HOME) + .withDescription( + "Path to the JRE/JDK. This should be installed at the same location on all cluster nodes ($JAVA_HOME, java.home by default)") + .create()); + + options.addOption( + OptionBuilder.hasArg().withArgName(OPTION_LLAP_PROPERTIES_FILE).withLongOpt(OPTION_LLAP_PROPERTIES_FILE) + .withDescription( + "Properties file containing llap properties. This will be overridden by --hiveconf") + .create()); + // -hiveconf x=y options.addOption(OptionBuilder.withValueSeparator().hasArgs(2).withArgName("property=value") - .withLongOpt("hiveconf").withDescription("Use value for given property").create()); + .withLongOpt(OPTION_HIVECONF) + .withDescription("Use value for given property. Overridden by explicit parameters") + .create()); // [-H|--help] options.addOption(new Option("H", "help", false, "Print help information")); @@ -166,7 +212,7 @@ private static long parseSuffixed(String value) { return StringUtils.TraditionalBinaryPrefix.string2long(value); } - public LlapOptions processOptions(String argv[]) throws ParseException { + public LlapOptions processOptions(String argv[]) throws ParseException, IOException { commandLine = new GnuParser().parse(options, argv); if (commandLine.hasOption('H') || false == commandLine.hasOption("instances")) { // needs at least --instances @@ -174,30 +220,41 @@ public LlapOptions processOptions(String argv[]) throws ParseException { return null; } - int instances = Integer.parseInt(commandLine.getOptionValue("instances")); - String directory = commandLine.getOptionValue("directory"); - String jars = commandLine.getOptionValue("auxjars"); + int instances = Integer.parseInt(commandLine.getOptionValue(OPTION_INSTANCES)); + String directory = commandLine.getOptionValue(OPTION_DIRECTORY); + String jars = commandLine.getOptionValue(OPTION_AUXJARS); - String name = commandLine.getOptionValue("name", null); + String name = commandLine.getOptionValue(OPTION_NAME, null); - final int executors = Integer.parseInt(commandLine.getOptionValue("executors", "-1")); - final long cache = parseSuffixed(commandLine.getOptionValue("cache", "-1")); - final long size = parseSuffixed(commandLine.getOptionValue("size", "-1")); - final long xmx = parseSuffixed(commandLine.getOptionValue("xmx", "-1")); - final boolean isHbase = Boolean.parseBoolean(commandLine.getOptionValue("auxhbase", "true")); + final int executors = Integer.parseInt(commandLine.getOptionValue(OPTION_EXECUTORS, "-1")); + final long cache = parseSuffixed(commandLine.getOptionValue(OPTION_CACHE, "-1")); + final long size = parseSuffixed(commandLine.getOptionValue(OPTION_SIZE, "-1")); + final long xmx = parseSuffixed(commandLine.getOptionValue(OPTION_XMX, "-1")); + final boolean isHbase = Boolean.parseBoolean(commandLine.getOptionValue(OPTION_AUXHBASE, "true")); final Properties hiveconf; - if (commandLine.hasOption("hiveconf")) { - hiveconf = commandLine.getOptionProperties("hiveconf"); + if (commandLine.hasOption(OPTION_HIVECONF)) { + hiveconf = commandLine.getOptionProperties(OPTION_HIVECONF); } else { hiveconf = new Properties(); } + final Properties llapProperties = new Properties(); + if (commandLine.hasOption(OPTION_LLAP_PROPERTIES_FILE)) { + String llapPropertiesFile = commandLine.getOptionValue(OPTION_LLAP_PROPERTIES_FILE); + InputStream inputStream = new FileInputStream(llapPropertiesFile); + llapProperties.load(inputStream); + } + + String javaHome = null; + if (commandLine.hasOption(OPTION_JAVA_HOME)) { + javaHome = commandLine.getOptionValue(OPTION_JAVA_HOME); + } // loglevel, chaosmonkey & args are parsed by the python processor return new LlapOptions( - name, instances, directory, executors, cache, size, xmx, jars, isHbase, hiveconf); + name, instances, directory, executors, cache, size, xmx, jars, isHbase, hiveconf, llapProperties, javaHome); } diff --git llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java index 0399798..9f50d4d 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java @@ -25,8 +25,10 @@ import java.util.Map.Entry; import java.util.Collection; +import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos; import org.apache.hadoop.hive.llap.tezplugins.LlapTezUtils; +import org.apache.tez.dag.api.TezConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -52,14 +54,19 @@ public class LlapServiceDriver { protected static final Logger LOG = LoggerFactory.getLogger(LlapServiceDriver.class.getName()); + private static final String[] DEFAULT_AUX_CLASSES = new String[] { "org.apache.hive.hcatalog.data.JsonSerDe" }; private static final String HBASE_SERDE_CLASS = "org.apache.hadoop.hive.hbase.HBaseSerDe"; private static final String[] NEEDED_CONFIGS = { - "tez-site.xml", "hive-site.xml", "llap-daemon-site.xml", "core-site.xml" }; + "tez-site.xml", "hive-site.xml", "core-site.xml", "yarn-site.xml" }; private static final String[] OPTIONAL_CONFIGS = { "ssl-server.xml" }; + /** + * This is a working configuration for the instance to merge various variables. + * It is not written out for llap server usage + */ private final Configuration conf; public LlapServiceDriver() { @@ -85,8 +92,10 @@ public static void main(String[] args) throws Exception { /** * Intersect llap-daemon-site.xml configuration properties against an existing Configuration * object, while resolving any ${} parameters that might be present. - * - * @param raw + * + * @param configured + * @param first + * @param resources * @return configuration object which is a slice of configured */ public static Configuration resolve(Configuration configured, String first, String... resources) { @@ -116,6 +125,7 @@ private void run(String[] args) throws Exception { return; } + // Working directory. Path tmpDir = new Path(options.getDirectory()); if (conf == null) { @@ -137,6 +147,27 @@ private void run(String[] args) throws Exception { } conf.reloadConfiguration(); + + // Set everything from the llap-specified-configuration first. Overwritten by hiveconf and individual parameters + for(Entry props : options.getPropertiesFromFile().entrySet()) { + String key = (String) props.getKey(); + if (HiveConf.getLlapDaemonConfVars().contains(key)) { + conf.set(key, (String) props.getValue()); + } else { + LOG.warn("Ignoring unknown llap server parameter: [{}]", key); + } + } + + // Set everything from hive-conf next. This will be overwritten by individual parameters. + for (Entry props : options.getConfig().entrySet()) { + String key = (String) props.getKey(); + if (HiveConf.getLlapDaemonConfVars().contains(key)) { + conf.set(key, (String) props.getValue()); + } else { + LOG.warn("Ignoring unknown llap server parameter: [{}]", key); + } + } + if (options.getName() != null) { // update service registry configs - caveat: this has nothing to do with the actual settings // as read by the AM @@ -160,9 +191,11 @@ private void run(String[] args) throws Exception { } } + // This parameter is read in package.py - and nowhere else. Does not need to be part of HiveConf - that's just confusing. final long minAlloc = conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, -1); + long containerSize = -1; if (options.getSize() != -1) { - final long containerSize = options.getSize() / (1024 * 1024); + containerSize = options.getSize() / (1024 * 1024); Preconditions.checkArgument(containerSize >= minAlloc, "Container size should be greater than minimum allocation(%s)", minAlloc + "m"); conf.setLong(ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB.varname, containerSize); @@ -186,10 +219,9 @@ private void run(String[] args) throws Exception { / (1024 * 1024)); } - for (Entry props : options.getConfig().entrySet()) { - conf.set((String) props.getKey(), (String) props.getValue()); - } + + // KKK Somehow consolidate this. The template file should be sufficient. URL logger = conf.getResource("llap-daemon-log4j2.properties"); if (null == logger) { @@ -205,9 +237,10 @@ private void run(String[] args) throws Exception { LOG.warn("Unable to find llap scripts:" + scripts); } + Path libDir = new Path(tmpDir, "lib"); - String tezLibs = conf.get("tez.lib.uris"); + String tezLibs = conf.get(TezConfiguration.TEZ_LIB_URIS); if (tezLibs == null) { LOG.warn("Missing tez.lib.uris in tez-site.xml"); } @@ -267,27 +300,40 @@ private void run(String[] args) throws Exception { Path confPath = new Path(tmpDir, "conf"); lfs.mkdirs(confPath); + // Copy over the mandatory configs for the package. for (String f : NEEDED_CONFIGS) { - copyConfig(options, lfs, confPath, f); + copyConfig(lfs, confPath, f); } for (String f : OPTIONAL_CONFIGS) { try { - copyConfig(options, lfs, confPath, f); + copyConfig(lfs, confPath, f); } catch (Throwable t) { LOG.info("Error getting an optional config " + f + "; ignoring: " + t.getMessage()); } } + createLlapDaemonConfig(lfs, confPath); lfs.copyFromLocalFile(new Path(logger.toString()), confPath); - String java_home = System.getenv("JAVA_HOME"); - String jre_home = System.getProperty("java.home"); - if (java_home == null) { - java_home = jre_home; - } else if (!java_home.equals(jre_home)) { - LOG.warn("Java versions might not match : JAVA_HOME=%s,process jre=%s", - java_home, jre_home); + String java_home; + if (options.getJavaPath() == null || options.getJavaPath().isEmpty()) { + java_home = System.getenv("JAVA_HOME"); + String jre_home = System.getProperty("java.home"); + if (java_home == null) { + java_home = jre_home; + } else if (!java_home.equals(jre_home)) { + // KKK Ensure a log4j2 configuration file is available for hive --service llap + LOG.warn("Java versions might not match : JAVA_HOME=%s,process jre=%s", + java_home, jre_home); + } + } else { + java_home = options.getJavaPath(); + } + if (java_home == null || java_home.isEmpty()) { + throw new RuntimeException( + "Could not determine JAVA_HOME from command line parameters, environment or system properties"); } + LOG.info("Using {} for JAVA_HOME", java_home); // extract configs for processing by the python fragments in Slider JSONObject configs = new JSONObject(); @@ -296,6 +342,7 @@ private void run(String[] args) throws Exception { configs.put(ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB.varname, HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB)); + configs.put(ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB.varname, containerSize); configs.put(HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname, HiveConf.getSizeVar(conf, HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE)); @@ -363,23 +410,25 @@ private void localizeJarForClass(FileSystem lfs, Path libDir, String className, } } - private void copyConfig( - LlapOptions options, FileSystem lfs, Path confPath, String f) throws IOException { - if (f.equals("llap-daemon-site.xml")) { - FSDataOutputStream confStream = lfs.create(new Path(confPath, f)); + private void createLlapDaemonConfig(FileSystem lfs, Path confPath) throws IOException { + FSDataOutputStream confStream = + lfs.create(new Path(confPath, LlapDaemonConfiguration.LLAP_DAEMON_SITE)); - Configuration copy = resolve(conf, "llap-daemon-site.xml"); - - for (Entry props : options.getConfig().entrySet()) { - // overrides - copy.set((String) props.getKey(), (String) props.getValue()); + // Set all the relevant LLAP-daemon configs here. + Configuration llapDaemonConf = new Configuration(false); + for (String confVar : HiveConf.getLlapDaemonConfVars()) { + String value = conf.get(confVar); + if (value != null) { + llapDaemonConf.set(confVar, value); } - - copy.writeXml(confStream); - confStream.close(); - } else { - // they will be file:// URLs - lfs.copyFromLocalFile(new Path(conf.getResource(f).toString()), confPath); } + + llapDaemonConf.writeXml(confStream); + confStream.close(); + } + + private void copyConfig(FileSystem lfs, Path confPath, String f) throws IOException { + // they will be file:// URLs + lfs.copyFromLocalFile(new Path(conf.getResource(f).toString()), confPath); } } diff --git llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java new file mode 100644 index 0000000..8948c7a --- /dev/null +++ llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java @@ -0,0 +1,32 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.llap.configuration; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; + +/** + * Configuration for LLAP daemon processes only. This should not be used by any clients. + */ +public class LlapDaemonConfiguration extends Configuration { + + @InterfaceAudience.Private + public static final String LLAP_DAEMON_SITE = "llap-daemon-site.xml"; + + public LlapDaemonConfiguration() { + super(false); + addResource(LLAP_DAEMON_SITE); + } +} \ No newline at end of file diff --git llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java index d1ec715..04c28cb 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java @@ -42,7 +42,6 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.LlapNodeId; -import org.apache.hadoop.hive.llap.configuration.LlapConfiguration; import org.apache.hadoop.hive.llap.daemon.QueryFailedHandler; import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol; import org.apache.hadoop.io.Text; diff --git llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index 94b3b41..0ec0cc6 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@ -29,7 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.llap.configuration.LlapConfiguration; +import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration; import org.apache.hadoop.hive.llap.daemon.ContainerRunner; import org.apache.hadoop.hive.llap.daemon.QueryFailedHandler; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto; @@ -293,7 +293,7 @@ public static void main(String[] args) throws Exception { try { // Cache settings will need to be setup in llap-daemon-site.xml - since the daemons don't read hive-site.xml // Ideally, these properties should be part of LlapDameonConf rather than HiveConf - LlapConfiguration daemonConf = new LlapConfiguration(); + LlapDaemonConfiguration daemonConf = new LlapDaemonConfiguration(); int numExecutors = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_NUM_EXECUTORS); String localDirList = HiveConf.getVar(daemonConf, ConfVars.LLAP_DAEMON_WORK_DIRS); diff --git llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java index 0676edd..0698c27 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java @@ -26,7 +26,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.llap.configuration.LlapConfiguration; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentSpecProto; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateProto; import org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler; @@ -40,7 +39,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantLock; diff --git llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java index deade5f..5fa2bf1 100644 --- llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java +++ llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java @@ -26,7 +26,6 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.llap.configuration.LlapConfiguration; import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.Service; diff --git llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java index 1cef218..a65bf5c 100644 --- llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java +++ llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java @@ -25,7 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.llap.configuration.LlapConfiguration; +import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration; import org.apache.hadoop.hive.llap.daemon.ContainerRunner; import org.apache.hadoop.hive.llap.protocol.LlapProtocolBlockingPB; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto; @@ -39,7 +39,7 @@ @Test(timeout = 10000) public void test() throws ServiceException, IOException { - LlapConfiguration daemonConf = new LlapConfiguration(); + LlapDaemonConfiguration daemonConf = new LlapDaemonConfiguration(); int rpcPort = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_RPC_PORT); int numHandlers = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_RPC_NUM_HANDLERS); ContainerRunner containerRunnerMock = mock(ContainerRunner.class);