diff --git common/src/java/org/apache/hadoop/hive/conf/Constants.java common/src/java/org/apache/hadoop/hive/conf/Constants.java new file mode 100644 index 0000000..00ec8c0 --- /dev/null +++ common/src/java/org/apache/hadoop/hive/conf/Constants.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hive.conf; + +public class Constants { + /* Constants for LLAP */ + public static final String LLAP_LOGGER_NAME_QUERY_ROUTING = "query-routing"; + public static final String LLAP_LOGGER_NAME_CONSOLE = "console"; + public static final String LLAP_LOGGER_NAME_RFA = "RFA"; +} diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 42f7d88..eca02c8 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -384,6 +384,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_WEB_SSL.varname); llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_CONTAINER_ID.varname); llapDaemonVarsSetLocal.add(ConfVars.LLAP_VALIDATE_ACLS.varname); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_LOGGER.varname); } /** @@ -2976,6 +2977,11 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "Whether to create the LLAP coordinator; since execution engine and container vs llap\n" + "settings are both coming from job configs, we don't know at start whether this should\n" + "be created. Default true."), + LLAP_DAEMON_LOGGER("hive.llap.daemon.logger", Constants.LLAP_LOGGER_NAME_RFA, + new StringSet(Constants.LLAP_LOGGER_NAME_QUERY_ROUTING, + Constants.LLAP_LOGGER_NAME_RFA, + Constants.LLAP_LOGGER_NAME_CONSOLE), + "logger used for llap-daemons."), SPARK_CLIENT_FUTURE_TIMEOUT("hive.spark.client.future.timeout", "60s", new TimeValidator(TimeUnit.SECONDS), diff --git llap-common/src/java/org/apache/hadoop/hive/llap/log/LogHelpers.java llap-common/src/java/org/apache/hadoop/hive/llap/log/LogHelpers.java new file mode 100644 index 0000000..e24354a --- /dev/null +++ llap-common/src/java/org/apache/hadoop/hive/llap/log/LogHelpers.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.llap.log; + +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.conf.Constants; + +@InterfaceAudience.Private +// If changing this file, make sure to make corresponding changes in llap-daemon-log4j2.properties +public class LogHelpers { + + public static final String LLAP_LOGGER_NAME_QUERY_ROUTING = + Constants.LLAP_LOGGER_NAME_QUERY_ROUTING; + public static final String LLAP_LOGGER_NAME_CONSOLE = Constants.LLAP_LOGGER_NAME_CONSOLE; + public static final String LLAP_LOGGER_NAME_RFA = Constants.LLAP_LOGGER_NAME_RFA; + +} diff --git llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java index f628ddf..f0a5b2f 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.llap.cli; +import com.google.common.base.Preconditions; import jline.TerminalFactory; import java.io.IOException; @@ -32,6 +33,7 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.llap.log.LogHelpers; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.util.StringUtils; @@ -56,6 +58,7 @@ // Options for the pythin script that are here because our option parser cannot ignore the unknown ones public static final String OPTION_ARGS = "args"; // forward as arg public static final String OPTION_LOGLEVEL = "loglevel"; // forward as arg + public static final String OPTION_LOGGER = "logger"; // forward as arg public static final String OPTION_CHAOS_MONKEY = "chaosmonkey"; // forward as arg public static final String OPTION_SLIDER_KEYTAB_DIR = "slider-keytab-dir"; public static final String OPTION_SLIDER_KEYTAB = "slider-keytab"; @@ -78,10 +81,12 @@ private final Properties conf; private final String javaPath; private final String llapQueueName; + private final String logger; public LlapOptions(String name, int instances, String directory, int executors, int ioThreads, long cache, long size, long xmx, String jars, boolean isHbase, - @Nonnull Properties hiveconf, String javaPath, String llapQueueName) + @Nonnull Properties hiveconf, String javaPath, String llapQueueName, + String logger) throws ParseException { if (instances <= 0) { throw new ParseException("Invalid configuration: " + instances @@ -100,6 +105,7 @@ public LlapOptions(String name, int instances, String directory, int executors, this.conf = hiveconf; this.javaPath = javaPath; this.llapQueueName = llapQueueName; + this.logger = logger; } public String getName() { @@ -153,6 +159,10 @@ public String getJavaPath() { public String getLlapQueueName() { return llapQueueName; } + + public String getLogger() { + return logger; + } } protected static final Logger l4j = LoggerFactory.getLogger(LlapOptionsProcessor.class.getName()); @@ -178,6 +188,12 @@ public LlapOptionsProcessor() { options.addOption(OptionBuilder.hasArg().withArgName(OPTION_LOGLEVEL).withLongOpt(OPTION_LOGLEVEL) .withDescription("log levels for the llap instance").create('l')); + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_LOGGER).withLongOpt(OPTION_LOGGER) + .withDescription( + "logger for llap instance ([" + LogHelpers.LLAP_LOGGER_NAME_RFA + "], " + + LogHelpers.LLAP_LOGGER_NAME_QUERY_ROUTING + ", " + LogHelpers.LLAP_LOGGER_NAME_CONSOLE) + .create()); + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_CHAOS_MONKEY).withLongOpt(OPTION_CHAOS_MONKEY) .withDescription("chaosmonkey interval").create('m')); @@ -284,10 +300,19 @@ public LlapOptions processOptions(String argv[]) throws ParseException, IOExcept javaHome = commandLine.getOptionValue(OPTION_JAVA_HOME); } + String logger = null; + if (commandLine.hasOption(OPTION_LOGGER)) { + logger = commandLine.getOptionValue(OPTION_LOGGER); + Preconditions.checkArgument( + logger.equalsIgnoreCase(LogHelpers.LLAP_LOGGER_NAME_QUERY_ROUTING) || + logger.equalsIgnoreCase(LogHelpers.LLAP_LOGGER_NAME_CONSOLE) || + logger.equalsIgnoreCase(LogHelpers.LLAP_LOGGER_NAME_RFA)); + } + // loglevel, chaosmonkey & args are parsed by the python processor return new LlapOptions(name, instances, directory, executors, ioThreads, cache, - size, xmx, jars, isHbase, hiveconf, javaHome, queueName); + size, xmx, jars, isHbase, hiveconf, javaHome, queueName, logger); } private void printUsage() { diff --git llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java index f41912b..9d366ac 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java @@ -199,6 +199,11 @@ private void run(String[] args) throws Exception { "@" + options.getName()); } + if (options.getLogger() != null) { + HiveConf.setVar(conf, ConfVars.LLAP_DAEMON_LOGGER, options.getLogger()); + propsDirectOptions.setProperty(ConfVars.LLAP_DAEMON_LOGGER.varname, options.getLogger()); + } + if (options.getSize() != -1) { if (options.getCache() != -1) { if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.LLAP_ALLOCATOR_MAPPED) == false) { @@ -572,6 +577,15 @@ private void localizeJarForClass(FileSystem lfs, Path libDir, String className, } } + /** + * + * @param lfs filesystem on which file will be generated + * @param confPath path wher the config will be generated + * @param configured the base configuration instances + * @param direct properties specified directly - i.e. using the properties exact option + * @param hiveconf properties specifried via --hiveconf + * @throws IOException + */ private void createLlapDaemonConfig(FileSystem lfs, Path confPath, Configuration configured, Properties direct, Properties hiveconf) throws IOException { FSDataOutputStream confStream = diff --git llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapConstants.java llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapConstants.java index cdb4070..112173c 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapConstants.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapConstants.java @@ -28,4 +28,5 @@ // Note: Do not change without changing the corresponding reference in llap-daemon-log4j2.properties public static final String LLAP_LOG4j2_PURGE_POLICY_NAME_QUERY_ROUTING = "llapLogPurgerQueryRouting"; + } diff --git llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java index e078fe1..f094039 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.llap.log.Log4jQueryCompleteMarker; +import org.apache.hadoop.hive.llap.log.LogHelpers; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.log4j.MDC; @@ -73,6 +74,7 @@ private final FileSystem localFs; private final String clusterId; private final long defaultDeleteDelaySeconds; + private final boolean routeBasedLoggingEnabled; // TODO At the moment there's no way of knowing whether a query is running or not. // A race is possible between dagComplete and registerFragment - where the registerFragment @@ -118,6 +120,16 @@ public QueryTracker(Configuration conf, String[] localDirsBase, String clusterId conf, ConfVars.LLAP_DAEMON_NUM_FILE_CLEANER_THREADS); this.executorService = Executors.newScheduledThreadPool(numCleanerThreads, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("QueryFileCleaner %d").build()); + + String logger = HiveConf.getVar(conf, ConfVars.LLAP_DAEMON_LOGGER); + if (logger != null && (logger.equalsIgnoreCase(LogHelpers.LLAP_LOGGER_NAME_QUERY_ROUTING))) { + routeBasedLoggingEnabled = true; + } else { + routeBasedLoggingEnabled = false; + } + LOG.info( + "QueryTracker setup with numCleanerThreads={}, defaultCleanupDelay(s)={}, routeBasedLogging={}", + numCleanerThreads, defaultDeleteDelaySeconds, routeBasedLoggingEnabled); } /** @@ -230,19 +242,21 @@ void fragmentComplete(QueryFragmentInfo fragmentInfo) { } } - // Inform the routing purgePolicy. - // Send out a fake log message at the ERROR level with the MDC for this query setup. With an - // LLAP custom appender this message will not be logged. - final String dagId = queryInfo.getDagIdString(); - final String queryId = queryInfo.getHiveQueryIdString(); - MDC.put("dagId", dagId); - MDC.put("queryId", queryId); - try { - LOG.error(QUERY_COMPLETE_MARKER, "Ignore this. Log line to interact with logger." + - " Query complete: " + queryInfo.getHiveQueryIdString() + ", " + - queryInfo.getDagIdString()); - } finally { - MDC.clear(); + if (routeBasedLoggingEnabled) { + // Inform the routing purgePolicy. + // Send out a fake log message at the ERROR level with the MDC for this query setup. With an + // LLAP custom appender this message will not be logged. + final String dagId = queryInfo.getDagIdString(); + final String queryId = queryInfo.getHiveQueryIdString(); + MDC.put("dagId", dagId); + MDC.put("queryId", queryId); + try { + LOG.error(QUERY_COMPLETE_MARKER, "Ignore this. Log line to interact with logger." + + " Query complete: " + queryInfo.getHiveQueryIdString() + ", " + + queryInfo.getDagIdString()); + } finally { + MDC.clear(); + } } // Clearing this before sending a kill is OK, since canFinish will change to false. diff --git llap-server/src/main/resources/llap-daemon-log4j2.properties llap-server/src/main/resources/llap-daemon-log4j2.properties index ffa0bf8..422a92a 100644 --- llap-server/src/main/resources/llap-daemon-log4j2.properties +++ llap-server/src/main/resources/llap-daemon-log4j2.properties @@ -14,6 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. + +# This is the log4j2 properties file used by llap-daemons. There's several loggers defined, which +# can be selected while configuring LLAP. +# Based on the one selected - UI links etc need to be manipulated in the system. +# Note: Some names and logic is common to this file and llap LogHelpers. Make sure to change that +# as well, if changing this file. + status = INFO name = LlapDaemonLog4j2 packages = org.apache.hadoop.hive.ql.log @@ -28,7 +35,7 @@ property.llap.daemon.log.maxfilesize = 256MB property.llap.daemon.log.maxbackupindex = 240 # list of all appenders -appenders = console, RFA, HISTORYAPPENDER, dag-routing, query-routing +appenders = console, RFA, HISTORYAPPENDER, query-routing # console appender appender.console.type = Console @@ -41,7 +48,7 @@ appender.console.layout.pattern = %d{ISO8601} %5p [%t (%X{fragmentId})] %c{2}: % appender.RFA.type = RollingRandomAccessFile appender.RFA.name = RFA appender.RFA.fileName = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.log.file} -appender.RFA.filePattern = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.log.file}_%d{yyyy-MM-dd-HH}_%i +appender.RFA.filePattern = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.log.file}_%d{yyyy-MM-dd-HH}_%i.done appender.RFA.layout.type = PatternLayout appender.RFA.layout.pattern = %d{ISO8601} %-5p [%t (%X{fragmentId})] %c: %m%n appender.RFA.policies.type = Policies @@ -57,7 +64,7 @@ appender.RFA.strategy.max = ${sys:llap.daemon.log.maxbackupindex} appender.HISTORYAPPENDER.type = RollingRandomAccessFile appender.HISTORYAPPENDER.name = HISTORYAPPENDER appender.HISTORYAPPENDER.fileName = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.historylog.file} -appender.HISTORYAPPENDER.filePattern = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.historylog.file}_%d{yyyy-MM-dd}_%i +appender.HISTORYAPPENDER.filePattern = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.historylog.file}_%d{yyyy-MM-dd}_%i.done appender.HISTORYAPPENDER.layout.type = PatternLayout appender.HISTORYAPPENDER.layout.pattern = %m%n appender.HISTORYAPPENDER.policies.type = Policies @@ -69,29 +76,6 @@ appender.HISTORYAPPENDER.policies.time.modulate = true appender.HISTORYAPPENDER.strategy.type = DefaultRolloverStrategy appender.HISTORYAPPENDER.strategy.max = ${sys:llap.daemon.log.maxbackupindex} -# dagId based routing file appender -appender.dag-routing.type = Routing -appender.dag-routing.name = dag-routing -appender.dag-routing.routes.type = Routes -appender.dag-routing.routes.pattern = $${ctx:dagId} -#Purge polciy for dag-based Routing Appender -appender.dag-routing.purgePolicy.type = LlapRoutingAppenderPurgePolicy -# Note: Do not change this name without changing the corresponding entry in LlapConstants -appender.dag-routing.purgePolicy.name = llapLogPurgerDagRouting -# default route -appender.dag-routing.routes.route-default.type = Route -appender.dag-routing.routes.route-default.key = $${ctx:dagId} -appender.dag-routing.routes.route-default.ref = RFA -# dagId based route -appender.dag-routing.routes.route-mdc.type = Route -appender.dag-routing.routes.route-mdc.file-mdc.type = LlapWrappedAppender -appender.dag-routing.routes.route-mdc.file-mdc.name = IrrelevantName-dag-routing -appender.dag-routing.routes.route-mdc.file-mdc.app.type=RandomAccessFile -appender.dag-routing.routes.route-mdc.file-mdc.app.name=file-mdc -appender.dag-routing.routes.route-mdc.file-mdc.app.fileName = ${sys:llap.daemon.log.dir}/${ctx:dagId}.log -appender.dag-routing.routes.route-mdc.file-mdc.app.layout.type = PatternLayout -appender.dag-routing.routes.route-mdc.file-mdc.app.layout.pattern = %d{ISO8601} %5p [%t (%X{fragmentId})] %c{2}: %m%n - # queryId based routing file appender appender.query-routing.type = Routing appender.query-routing.name = query-routing diff --git llap-server/src/main/resources/package.py llap-server/src/main/resources/package.py index a200414..cf22480 100644 --- llap-server/src/main/resources/package.py +++ llap-server/src/main/resources/package.py @@ -71,7 +71,7 @@ def main(args): parser.add_argument("--args", default="") parser.add_argument("--name", default="llap0") parser.add_argument("--loglevel", default="INFO") - parser.add_argument("--logger", default="query-routing") + parser.add_argument("--logger", default="RFA") parser.add_argument("--chaosmonkey", type=int, default=0) parser.add_argument("--slider-am-container-mb", type=int, default=1024) parser.add_argument("--slider-keytab-dir", default="") diff --git llap-server/src/main/resources/templates.py llap-server/src/main/resources/templates.py index 505219a..95e6a4a 100644 --- llap-server/src/main/resources/templates.py +++ llap-server/src/main/resources/templates.py @@ -103,6 +103,7 @@ "metadata" : { }, "global" : { + "yarn.log.include.patterns": ".*\\\\.done" }, "components": { "slider-appmaster": {