diff --git a/beeline/src/main/resources/beeline-log4j2.properties b/beeline/src/main/resources/beeline-log4j2.properties new file mode 100644 index 0000000..103d722 --- /dev/null +++ b/beeline/src/main/resources/beeline-log4j2.properties @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = BeelineLog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.hive.log.level = WARN +property.hive.root.logger = console + +# list of all appenders +appenders = console + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + +# list of all loggers +loggers = HiveConnection + +# HiveConnection logs useful info for dynamic service discovery +logger.HiveConnection.name = org.apache.hive.jdbc.HiveConnection +logger.HiveConnection.level = INFO + +# root logger +rootLogger.level = ${sys:hive.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} diff --git a/beeline/src/main/resources/beeline-log4j2.xml b/beeline/src/main/resources/beeline-log4j2.xml deleted file mode 100644 index 55ec0f5..0000000 --- a/beeline/src/main/resources/beeline-log4j2.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - WARN - console - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/bin/ext/beeline.sh b/bin/ext/beeline.sh index 9de8f6c..90e562f 100644 --- a/bin/ext/beeline.sh +++ b/bin/ext/beeline.sh @@ -31,7 +31,7 @@ beeline () { hadoopClasspath="${HADOOP_CLASSPATH}:" fi export HADOOP_CLASSPATH="${hadoopClasspath}${HIVE_CONF_DIR}:${beelineJarPath}:${superCsvJarPath}:${jlineJarPath}:${jdbcStandaloneJarPath}" - export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=beeline-log4j2.xml " + export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=beeline-log4j2.properties " exec $HADOOP jar ${beelineJarPath} $CLASS $HIVE_OPTS "$@" } diff --git a/bin/hive b/bin/hive index 47d99f7..d7c1353 100755 --- a/bin/hive +++ b/bin/hive @@ -194,7 +194,7 @@ if [ "$HADOOP_HOME" == "" ]; then fi # to avoid errors from log4j2 automatic configuration loading -export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=hive-log4j2.xml " +export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=hive-log4j2.properties " HADOOP=$HADOOP_HOME/bin/hadoop if [ ! -f ${HADOOP} ]; then diff --git a/common/src/java/org/apache/hadoop/hive/common/LogUtils.java b/common/src/java/org/apache/hadoop/hive/common/LogUtils.java index 3be8733..adcf805 100644 --- a/common/src/java/org/apache/hadoop/hive/common/LogUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/LogUtils.java @@ -32,8 +32,8 @@ */ public class LogUtils { - private static final String HIVE_L4J = "hive-log4j2.xml"; - private static final String HIVE_EXEC_L4J = "hive-exec-log4j2.xml"; + private static final String HIVE_L4J = "hive-log4j2.properties"; + private static final String HIVE_EXEC_L4J = "hive-exec-log4j2.properties"; private static final Logger l4j = LoggerFactory.getLogger(LogUtils.class); @SuppressWarnings("serial") diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 4f8209a..f930aa1 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1744,12 +1744,12 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { // logging configuration HIVE_LOG4J_FILE("hive.log4j.file", "", "Hive log4j configuration file.\n" + - "If the property is not set, then logging will be initialized using hive-log4j2.xml found on the classpath.\n" + + "If the property is not set, then logging will be initialized using hive-log4j2.properties found on the classpath.\n" + "If the property is set, the value must be a valid URI (java.net.URI, e.g. \"file:///tmp/my-logging.xml\"), \n" + "which you can then extract a URL from and pass to PropertyConfigurator.configure(URL)."), HIVE_EXEC_LOG4J_FILE("hive.exec.log4j.file", "", "Hive log4j configuration file for execution mode(sub command).\n" + - "If the property is not set, then logging will be initialized using hive-exec-log4j2.xml found on the classpath.\n" + + "If the property is not set, then logging will be initialized using hive-exec-log4j2.properties found on the classpath.\n" + "If the property is set, the value must be a valid URI (java.net.URI, e.g. \"file:///tmp/my-logging.xml\"), \n" + "which you can then extract a URL from and pass to PropertyConfigurator.configure(URL)."), diff --git a/common/src/main/resources/hive-log4j.properties b/common/src/main/resources/hive-log4j.properties deleted file mode 100644 index 7e62c71..0000000 --- a/common/src/main/resources/hive-log4j.properties +++ /dev/null @@ -1,88 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Define some default values that can be overridden by system properties -hive.log.threshold=ALL -hive.root.logger=INFO,DRFA -hive.log.dir=${java.io.tmpdir}/${user.name} -hive.log.file=hive.log - -# Define the root logger to the system property "hadoop.root.logger". -log4j.rootLogger=${hive.root.logger}, EventCounter - -# Logging Threshold -log4j.threshold=${hive.log.threshold} - -# -# Daily Rolling File Appender -# -# Use the PidDailyerRollingFileAppend class instead if you want to use separate log files -# for different CLI session. -# -# log4j.appender.DRFA=org.apache.hadoop.hive.ql.log.PidDailyRollingFileAppender - -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender - -log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file} - -# Rollver at midnight -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd - -# 30-day backup -#log4j.appender.DRFA.MaxBackupIndex=30 -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout - -# Pattern format: Date LogLevel LoggerName LogMessage -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -# Debugging Pattern format -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t(%x)]: %c{2} (%F:%M(%L)) - %m%n - - -# -# console -# Add "console" to rootlogger above if you want to use this -# - -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t(%x)]: %p %c{2}: %m%n -log4j.appender.console.encoding=UTF-8 - -#custom logging levels -#log4j.logger.xxx=DEBUG - -# -# Event Counter Appender -# Sends counts of logging messages at different severity levels to Hadoop Metrics. -# -log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter - - -log4j.category.DataNucleus=ERROR,DRFA -log4j.category.Datastore=ERROR,DRFA -log4j.category.Datastore.Schema=ERROR,DRFA -log4j.category.JPOX.Datastore=ERROR,DRFA -log4j.category.JPOX.Plugin=ERROR,DRFA -log4j.category.JPOX.MetaData=ERROR,DRFA -log4j.category.JPOX.Query=ERROR,DRFA -log4j.category.JPOX.General=ERROR,DRFA -log4j.category.JPOX.Enhancer=ERROR,DRFA - - -# Silence useless ZK logs -log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA -log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA diff --git a/common/src/main/resources/hive-log4j2.properties b/common/src/main/resources/hive-log4j2.properties new file mode 100644 index 0000000..12cd9ac --- /dev/null +++ b/common/src/main/resources/hive-log4j2.properties @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = HiveLog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.hive.log.level = INFO +property.hive.root.logger = DRFA +property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} +property.hive.log.file = hive.log + +# list of all appenders +appenders = console, DRFA + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + +# daily rolling file appender +appender.DRFA.type = RollingFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file} +# Use %pid in the filePattern to append @ to the filename if you want separate log files for different CLI session +appender.DRFA.filePattern = ${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd} +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = 30 + +# list of all loggers +loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX + +logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn +logger.NIOServerCnxn.level = WARN + +logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO +logger.ClientCnxnSocketNIO.level = WARN + +logger.DataNucleus.name = DataNucleus +logger.DataNucleus.level = ERROR + +logger.Datastore.name = Datastore +logger.Datastore.level = ERROR + +logger.JPOX.name = JPOX +logger.JPOX.level = ERROR + +# root logger +rootLogger.level = ${sys:hive.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} diff --git a/common/src/main/resources/hive-log4j2.xml b/common/src/main/resources/hive-log4j2.xml deleted file mode 100644 index 50b03b5..0000000 --- a/common/src/main/resources/hive-log4j2.xml +++ /dev/null @@ -1,110 +0,0 @@ - - - - - - - INFO - DRFA - ${sys:java.io.tmpdir}/${sys:user.name} - hive.log - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java b/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java index 92269e7..fd6eda2 100644 --- a/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java +++ b/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java @@ -17,9 +17,7 @@ */ package org.apache.hadoop.hive.conf; -import java.io.BufferedReader; import java.io.File; -import java.io.InputStreamReader; import org.apache.hadoop.hive.common.LogUtils; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -34,13 +32,8 @@ * Loads configuration files located in common/src/test/resources. */ public class TestHiveLogging extends TestCase { - private final Runtime runTime; - private Process process; - public TestHiveLogging() { super(); - runTime = Runtime.getRuntime(); - process = null; } private void configLog(String hiveLog4jTest, String hiveExecLog4jTest) @@ -57,25 +50,6 @@ private void configLog(String hiveLog4jTest, String hiveExecLog4jTest) assertEquals(expectedLog4jExecPath, conf.getVar(ConfVars.HIVE_EXEC_LOG4J_FILE)); } - private void runCmd(String cmd) throws Exception { - process = runTime.exec(cmd); - process.waitFor(); - } - - private void getCmdOutput(String logFile) throws Exception { - boolean logCreated = false; - BufferedReader buf = new BufferedReader( - new InputStreamReader(process.getInputStream())); - String line = ""; - while((line = buf.readLine()) != null) { - line = line.replace("//", "/"); - if (line.equals(logFile)) { - logCreated = true; - } - } - assertEquals(true, logCreated); - } - public void cleanLog(File logFile) { if (logFile.exists()) { logFile.delete(); @@ -107,6 +81,6 @@ public void testHiveLogging() throws Exception { String customLogName = "hiveLog4j2Test.log"; File customLogFile = new File(customLogPath, customLogName); RunTest(customLogFile, - "hive-log4j2-test.xml", "hive-exec-log4j2-test.xml"); + "hive-log4j2-test.properties", "hive-exec-log4j2-test.properties"); } } diff --git a/common/src/test/resources/hive-exec-log4j2-test.properties b/common/src/test/resources/hive-exec-log4j2-test.properties new file mode 100644 index 0000000..9afa5fe --- /dev/null +++ b/common/src/test/resources/hive-exec-log4j2-test.properties @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = HiveExecLog4j2Test +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.hive.log.level = INFO +property.hive.root.logger = FA +property.hive.log.dir = ${sys:test.tmp.dir}/${sys:user.name}-TestHiveLogging +property.hive.log.file = hiveExecLog4j2Test.log + +# list of all appenders +appenders = console, FA + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + +# simple file appender +appender.FA.type = File +appender.FA.name = FA +appender.FA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file} +appender.FA.layout.type = PatternLayout +appender.FA.layout.pattern = %d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + +# list of all loggers +loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX + +logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn +logger.NIOServerCnxn.level = WARN + +logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO +logger.ClientCnxnSocketNIO.level = WARN + +logger.DataNucleus.name = DataNucleus +logger.DataNucleus.level = ERROR + +logger.Datastore.name = Datastore +logger.Datastore.level = ERROR + +logger.JPOX.name = JPOX +logger.JPOX.level = ERROR + +# root logger +rootLogger.level = ${sys:hive.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} diff --git a/common/src/test/resources/hive-exec-log4j2-test.xml b/common/src/test/resources/hive-exec-log4j2-test.xml deleted file mode 100644 index 03487de..0000000 --- a/common/src/test/resources/hive-exec-log4j2-test.xml +++ /dev/null @@ -1,85 +0,0 @@ - - - - - - - INFO - FA - ${sys:test.tmp.dir}/${sys:user.name}-TestHiveLogging - hiveExecLog4j2Test.log - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/common/src/test/resources/hive-log4j2-test.properties b/common/src/test/resources/hive-log4j2-test.properties new file mode 100644 index 0000000..ca69a8f --- /dev/null +++ b/common/src/test/resources/hive-log4j2-test.properties @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = HiveLog4j2Test +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.hive.log.level = WARN +property.hive.root.logger = DRFA +property.hive.log.dir = ${sys:test.tmp.dir}/${sys:user.name}-TestHiveLogging +property.hive.log.file = hiveLog4j2Test.log + +# list of all appenders +appenders = console, DRFA + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + +# daily rolling file appender +appender.DRFA.type = RollingFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file} +appender.DRFA.filePattern = ${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd} +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = 30 + +# list of all loggers +loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX + +logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn +logger.NIOServerCnxn.level = WARN + +logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO +logger.ClientCnxnSocketNIO.level = WARN + +logger.DataNucleus.name = DataNucleus +logger.DataNucleus.level = ERROR + +logger.Datastore.name = Datastore +logger.Datastore.level = ERROR + +logger.JPOX.name = JPOX +logger.JPOX.level = ERROR + +# root logger +rootLogger.level = ${sys:hive.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} diff --git a/common/src/test/resources/hive-log4j2-test.xml b/common/src/test/resources/hive-log4j2-test.xml deleted file mode 100644 index 0297e88..0000000 --- a/common/src/test/resources/hive-log4j2-test.xml +++ /dev/null @@ -1,94 +0,0 @@ - - - - - - - WARN - DRFA - ${sys:test.tmp.dir}/${sys:user.name}-TestHiveLogging - hiveLog4j2Test.log - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/data/conf/hive-log4j2.properties b/data/conf/hive-log4j2.properties new file mode 100644 index 0000000..90bec21 --- /dev/null +++ b/data/conf/hive-log4j2.properties @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = HiveLog4j2Test +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.hive.log.level = DEBUG +property.hive.root.logger = DRFA +property.hive.log.dir = ${sys:test.tmp.dir}/log +property.hive.log.file = hive.log + +# list of all appenders +appenders = console, DRFA + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + +# daily rolling file appender +appender.DRFA.type = RollingFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file} +appender.DRFA.filePattern = ${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd} +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = 30 + +# list of all loggers +loggers = HadoopIPC, HadoopSecurity, Hdfs, HdfsServer, HadoopMetrics2, Mortbay, Yarn, YarnServer, Tez, HadoopConf, Zookeeper, ServerCnxn, NIOServerCnxn, ClientCnxn, ClientCnxnSocket, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, Operator, Serde2Lazy, ObjectStore + +logger.HadoopIPC.name = org.apache.hadoop.ipc +logger.HadoopIPC.level = WARN + +logger.HadoopSecurity.name = org.apache.hadoop.security +logger.HadoopSecurity.level = INFO + +logger.Hdfs.name = org.apache.hadoop.hdfs +logger.Hdfs.level = INFO + +logger.HdfsServer.name = org.apache.hadoop.hdfs.server +logger.HdfsServer.level = WARN + +logger.HadoopMetrics2.name = org.apache.hadoop.metrics2 +logger.HadoopMetrics2.level = INFO + +logger.Mortbay.name = org.mortbay +logger.Mortbay.level = INFO + +logger.Yarn.name = org.apache.hadoop.yarn +logger.Yarn.level = INFO + +logger.YarnServer.name = org.apache.hadoop.yarn.server +logger.YarnServer.level = WARN + +logger.Tez.name = org.apache.tez +logger.Tez.level = INFO + +logger.HadoopConf.name = org.apache.hadoop.conf.Configuration +logger.HadoopConf.level = ERROR + +logger.Zookeeper.name = org.apache.zookeeper +logger.Zookeeper.level = INFO + +logger.ServerCnxn.name = org.apache.zookeeper.server.ServerCnxn +logger.ServerCnxn.level = WARN + +logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn +logger.NIOServerCnxn.level = WARN + +logger.ClientCnxn.name = org.apache.zookeeper.ClientCnxn +logger.ClientCnxn.level = WARN + +logger.ClientCnxnSocket.name = org.apache.zookeeper.ClientCnxnSocket +logger.ClientCnxnSocket.level = WARN + +logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO +logger.ClientCnxnSocketNIO.level = WARN + +logger.DataNucleus.name = DataNucleus +logger.DataNucleus.level = ERROR + +logger.Datastore.name = Datastore +logger.Datastore.level = ERROR + +logger.JPOX.name = JPOX +logger.JPOX.level = ERROR + +logger.Operator.name = org.apache.hadoop.hive.ql.exec.Operator +logger.Operator.level = INFO + +logger.Serde2Lazy.name = org.apache.hadoop.hive.serde2.lazy +logger.Serde2Lazy.level = INFO + +logger.ObjectStore.name = org.apache.hadoop.hive.metastore.ObjectStore +logger.ObjectStore.level = INFO + +# root logger +rootLogger.level = ${sys:hive.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} diff --git a/data/conf/hive-log4j2.xml b/data/conf/hive-log4j2.xml deleted file mode 100644 index 860d756..0000000 --- a/data/conf/hive-log4j2.xml +++ /dev/null @@ -1,170 +0,0 @@ - - - - - - - DEBUG - DRFA - ${sys:test.tmp.dir}/log - hive.log - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/data/conf/llap/hive-site.xml b/data/conf/llap/hive-site.xml index becb5b2..8f4f944 100644 --- a/data/conf/llap/hive-site.xml +++ b/data/conf/llap/hive-site.xml @@ -255,12 +255,12 @@ hive.tez.java.opts - -Dlog4j.configurationFile=tez-container-log4j2.xml -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA + -Dlog4j.configurationFile=tez-container-log4j2.properties -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA tez.am.launch.cmd-opts - -Dlog4j.configurationFile=tez-container-log4j2.xml -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA + -Dlog4j.configurationFile=tez-container-log4j2.properties -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA diff --git a/data/conf/spark/log4j2.properties b/data/conf/spark/log4j2.properties new file mode 100644 index 0000000..81f067d --- /dev/null +++ b/data/conf/spark/log4j2.properties @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = SparkLog4j2Test +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.spark.log.level = DEBUG +property.spark.root.logger = DRFA +property.spark.log.dir = target/tmp/log +property.spark.log.file = spark.log + +# list of all appenders +appenders = console, DRFA + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + +# daily rolling file appender +appender.DRFA.type = RollingFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${sys:spark.log.dir}/${sys:spark.log.file} +appender.DRFA.filePattern = ${sys:spark.log.dir}/${sys:spark.log.file}.%d{yyyy-MM-dd} +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = 30 + +# list of all loggers +loggers = SparkIMain, SparkILoop, Jetty, AbstractLifeCycle + +logger.SparkIMain.name = org.apache.spark.repl.SparkIMain$exprTyper +logger.SparkIMain.level = INFO + +logger.SparkILoop.name = org.apache.spark.repl.SparkILoop$SparkILoopInterpreter +logger.SparkILoop.level = INFO + +logger.Jetty.name = org.eclipse.jetty +logger.Jetty.level = WARN + +logger.AbstractLifeCycle.name = org.eclipse.jetty.util.component.AbstractLifeCycle +logger.AbstractLifeCycle.level = ERROR + +# root logger +rootLogger.level = ${sys:spark.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:spark.root.logger} diff --git a/data/conf/spark/log4j2.xml b/data/conf/spark/log4j2.xml deleted file mode 100644 index 395a2bf..0000000 --- a/data/conf/spark/log4j2.xml +++ /dev/null @@ -1,74 +0,0 @@ - - - - - - - DEBUG - DRFA - target/tmp/log - spark.log - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/data/conf/tez/hive-site.xml b/data/conf/tez/hive-site.xml index d008ad1..19c4f04 100644 --- a/data/conf/tez/hive-site.xml +++ b/data/conf/tez/hive-site.xml @@ -254,12 +254,12 @@ hive.tez.java.opts - -Dlog4j.configurationFile=tez-container-log4j2.xml -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA + -Dlog4j.configurationFile=tez-container-log4j2.properties -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA tez.am.launch.cmd-opts - -Dlog4j.configurationFile=tez-container-log4j2.xml -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA + -Dlog4j.configurationFile=tez-container-log4j2.properties -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA diff --git a/docs/xdocs/language_manual/cli.xml b/docs/xdocs/language_manual/cli.xml index eb91e44..aaa8e81 100644 --- a/docs/xdocs/language_manual/cli.xml +++ b/docs/xdocs/language_manual/cli.xml @@ -163,7 +163,7 @@ Sample Usage:

-Hive uses log4j for logging. These logs are not emitted to the standard output by default but are instead captured to a log file specified by Hive's log4j properties file. By default Hive will use hive-log4j2.xml in the conf/ directory of the hive installation which writes out logs to /tmp/$USER/hive.log and uses the WARN level. +Hive uses log4j for logging. These logs are not emitted to the standard output by default but are instead captured to a log file specified by Hive's log4j properties file. By default Hive will use hive-log4j2.properties in the conf/ directory of the hive installation which writes out logs to /tmp/$USER/hive.log and uses the WARN level.

It is often desirable to emit the logs to the standard output and/or change the logging level for debugging purposes. These can be done from the command line as follows:

diff --git a/hcatalog/bin/hcat_server.sh b/hcatalog/bin/hcat_server.sh index d1aecb8..7a7a45e 100644 --- a/hcatalog/bin/hcat_server.sh +++ b/hcatalog/bin/hcat_server.sh @@ -84,7 +84,7 @@ function start_hcat() { export AUX_CLASSPATH=${AUX_CLASSPATH} export HADOOP_HOME=$HADOOP_HOME - #export HADOOP_OPTS="-Dlog4j.configurationFile=file://${HCAT_PREFIX}/conf/log4j2.xml" + #export HADOOP_OPTS="-Dlog4j.configurationFile=file://${HCAT_PREFIX}/conf/log4j2.properties" export HADOOP_OPTS="${HADOOP_OPTS} -server -XX:+UseConcMarkSweepGC -XX:ErrorFile=${HCAT_LOG_DIR}/hcat_err_pid%p.log -Xloggc:${HCAT_LOG_DIR}/hcat_gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps" export HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-2048} # 8G is better if you have it export METASTORE_PORT=${METASTORE_PORT:-9083} diff --git a/hcatalog/bin/templeton.cmd b/hcatalog/bin/templeton.cmd index 759f654..1b1ae6f 100644 --- a/hcatalog/bin/templeton.cmd +++ b/hcatalog/bin/templeton.cmd @@ -59,7 +59,7 @@ setlocal enabledelayedexpansion if not defined TEMPLETON_LOG4J ( @rem must be prefixed with file: otherwise config is not picked up - set TEMPLETON_LOG4J=file:%WEBHCAT_CONF_DIR%\webhcat-log4j2.xml + set TEMPLETON_LOG4J=file:%WEBHCAT_CONF_DIR%\webhcat-log4j2.properties ) set TEMPLETON_OPTS=-Dtempleton.log.dir=%TEMPLETON_LOG_DIR% -Dlog4j.configurationFile=%TEMPLETON_LOG4J% %HADOOP_OPTS% set arguments=%JAVA_HEAP_MAX% %TEMPLETON_OPTS% -classpath %CLASSPATH% org.apache.hive.hcatalog.templeton.Main diff --git a/hcatalog/scripts/hcat_server_start.sh b/hcatalog/scripts/hcat_server_start.sh index 872d1b5..27504de 100755 --- a/hcatalog/scripts/hcat_server_start.sh +++ b/hcatalog/scripts/hcat_server_start.sh @@ -70,7 +70,7 @@ export AUX_CLASSPATH=${AUX_CLASSPATH} export HADOOP_HOME=$HADOOP_HOME -#export HADOOP_OPTS="-Dlog4j.configurationFile=file://${ROOT}/conf/log4j2.xml" +#export HADOOP_OPTS="-Dlog4j.configurationFile=file://${ROOT}/conf/log4j2.properties" export HADOOP_OPTS="${HADOOP_OPTS} -server -XX:+UseConcMarkSweepGC -XX:ErrorFile=${ROOT}/var/log/hcat_err_pid%p.log -Xloggc:${ROOT}/var/log/hcat_gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps" export HADOOP_HEAPSIZE=2048 # 8G is better if you have it diff --git a/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml b/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml index 6385e409..296b631 100644 --- a/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml +++ b/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml @@ -66,7 +66,7 @@ ${env.PIG_HOME}/bin/pig uncertainty.

Note: The location of the log files created by Templeton and some other properties - of the logging system are set in the webhcat-log4j2.xml file.

+ of the logging system are set in the webhcat-log4j2.properties file.

Variables diff --git a/hcatalog/src/docs/src/documentation/content/xdocs/install.xml b/hcatalog/src/docs/src/documentation/content/xdocs/install.xml index e2953a9..4ae0401 100644 --- a/hcatalog/src/docs/src/documentation/content/xdocs/install.xml +++ b/hcatalog/src/docs/src/documentation/content/xdocs/install.xml @@ -241,7 +241,7 @@

Server activity logs are located in root/var/log/hcat_server. Logging configuration is located at - root/conf/log4j2.xml. Server logging uses + root/conf/log4j2.properties. Server logging uses DailyRollingFileAppender by default. It will generate a new file per day and does not expire old log files automatically.

diff --git a/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.properties b/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.properties new file mode 100644 index 0000000..c618297 --- /dev/null +++ b/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.properties @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = HCatE2ELog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.hive.log.level = DEBUG +property.hive.root.logger = DRFA +property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} +property.hive.log.file = hive.log + +# list of all appenders +appenders = console, DRFA + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + +# daily rolling file appender +appender.DRFA.type = RollingFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file} +appender.DRFA.filePattern = ${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd} +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = 30 + +# list of all loggers +loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX + +logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn +logger.NIOServerCnxn.level = WARN + +logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO +logger.ClientCnxnSocketNIO.level = WARN + +logger.DataNucleus.name = DataNucleus +logger.DataNucleus.level = ERROR + +logger.Datastore.name = Datastore +logger.Datastore.level = ERROR + +logger.JPOX.name = JPOX +logger.JPOX.level = ERROR + +# root logger +rootLogger.level = ${sys:hive.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} diff --git a/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.xml b/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.xml deleted file mode 100644 index 87e18e2..0000000 --- a/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.xml +++ /dev/null @@ -1,110 +0,0 @@ - - - - - - - DEBUG - DRFA - ${sys:java.io.tmpdir}/${sys:user.name} - hive.log - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/hcatalog/src/test/e2e/templeton/deployers/start_hive_services.sh b/hcatalog/src/test/e2e/templeton/deployers/start_hive_services.sh index e59177c..66c0cb5 100755 --- a/hcatalog/src/test/e2e/templeton/deployers/start_hive_services.sh +++ b/hcatalog/src/test/e2e/templeton/deployers/start_hive_services.sh @@ -31,7 +31,7 @@ cp ${PROJ_HOME}/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site. #cp ${PROJ_HOME}/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.mssql.xml ${HIVE_HOME}/conf/hive-site.xml cp ${PROJ_HOME}/hcatalog/src/test/e2e/templeton/deployers/config/webhcat/webhcat-site.xml ${HIVE_HOME}/hcatalog/etc/webhcat/webhcat-site.xml -cp ${PROJ_HOME}/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.xml ${HIVE_HOME}/conf/hive-log4j2.xml +cp ${PROJ_HOME}/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.properties ${HIVE_HOME}/conf/hive-log4j2.properties if [ -f ${MYSQL_CLIENT_JAR} ]; then cp ${MYSQL_CLIENT_JAR} ${HIVE_HOME}/lib diff --git a/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh b/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh index c80fdd5..ec7c492 100644 --- a/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh +++ b/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh @@ -215,7 +215,7 @@ else fi if [[ -z "$WEBHCAT_LOG4J" ]]; then - WEBHCAT_LOG4J="file://$base_dir/etc/webhcat/webhcat-log4j2.xml"; + WEBHCAT_LOG4J="file://$base_dir/etc/webhcat/webhcat-log4j2.properties"; fi export HADOOP_USER_CLASSPATH_FIRST=true diff --git a/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.properties b/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.properties new file mode 100644 index 0000000..3ea2c7b --- /dev/null +++ b/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.properties @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = WebhcatLog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.webhcat.log.level = INFO +property.webhcat.root.logger = standard +property.webhcat.log.dir = . +property.webhcat.log.file = webhcat.log + +# list of all appenders +appenders = console, standard + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + +# daily rolling file appender +appender.standard.type = RollingFile +appender.standard.name = standard +appender.standard.fileName = ${sys:webhcat.log.dir}/${sys:webhcat.log.file} +appender.standard.filePattern = ${sys:webhcat.log.dir}/${sys:webhcat.log.file}.%d{yyyy-MM-dd} +appender.standard.layout.type = PatternLayout +appender.standard.layout.pattern = %-5p | %d{DATE} | %c | %m%n +appender.standard.policies.type = Policies +appender.standard.policies.time.type = TimeBasedTriggeringPolicy +appender.standard.policies.time.interval = 1 +appender.standard.policies.time.modulate = true +appender.standard.strategy.type = DefaultRolloverStrategy +appender.standard.strategy.max = 30 + +# list of all loggers +loggers = Jersey, WebComponent, Hadoop, HadoopConf, Zookeeper, Jetty + +logger.Jersey.name = com.sun.jersey +logger.Jersey.level = DEBUG + +logger.WebComponent.name = com.sun.jersey.spi.container.servlet.WebComponent +logger.WebComponent.level = ERROR + +logger.Hadoop.name = org.apache.hadoop +logger.Hadoop.level = INFO + +logger.HadoopConf.name = org.apache.hadoop.conf +logger.HadoopConf.level = WARN + +logger.Zookeeper.name = org.apache.zookeeper +logger.Zookeeper.level = WARN + +logger.Jetty.name = org.eclipse.jetty +logger.Jetty.level = INFO + +# root logger +rootLogger.level = ${sys:webhcat.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:webhcat.root.logger} diff --git a/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml b/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml deleted file mode 100644 index ef8e73d..0000000 --- a/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml +++ /dev/null @@ -1,74 +0,0 @@ - - - - - - - INFO - standard - . - webhcat.log - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java b/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java index 3a80eca..84193da 100644 --- a/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java +++ b/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java @@ -766,7 +766,7 @@ Integer init(String[] args) throws Exception { return 1; } // specify the default log4j2 properties file. - System.setProperty("log4j.configurationFile", "hive-log4j2.xml"); + System.setProperty("log4j.configurationFile", "hive-log4j2.properties"); conf = new Conf(); conf.init(); conn = new Conn(this); diff --git a/itests/pom.xml b/itests/pom.xml index 0686f1f..8a334dd 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -89,7 +89,7 @@ } mkdir -p $DOWNLOAD_DIR download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop2-without-hive.tgz" "spark" - cp -f $HIVE_ROOT/data/conf/spark/log4j2.xml $BASE_DIR/spark/conf/ + cp -f $HIVE_ROOT/data/conf/spark/log4j2.properties $BASE_DIR/spark/conf/ sed '/package /d' ${basedir}/${hive.path.to.root}/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleAdd.java > /tmp/UDFExampleAdd.java javac -cp ${settings.localRepository}/org/apache/hive/hive-exec/${project.version}/hive-exec-${project.version}.jar /tmp/UDFExampleAdd.java -d /tmp jar -cf /tmp/udfexampleadd-1.0.jar -C /tmp UDFExampleAdd.class diff --git a/packaging/src/main/assembly/bin.xml b/packaging/src/main/assembly/bin.xml index 742b3db..01d8394 100644 --- a/packaging/src/main/assembly/bin.xml +++ b/packaging/src/main/assembly/bin.xml @@ -295,7 +295,7 @@ 644 webhcat-default.xml - webhcat-log4j2.xml + webhcat-log4j2.properties true hcatalog/etc/webhcat @@ -343,19 +343,19 @@ - ${project.parent.basedir}/common/src/main/resources/hive-log4j2.xml + ${project.parent.basedir}/common/src/main/resources/hive-log4j2.properties conf - hive-log4j2.xml.template + hive-log4j2.properties.template - ${project.parent.basedir}/ql/src/main/resources/hive-exec-log4j2.xml + ${project.parent.basedir}/ql/src/main/resources/hive-exec-log4j2.properties conf - hive-exec-log4j2.xml.template + hive-exec-log4j2.properties.template - ${project.parent.basedir}/beeline/src/main/resources/beeline-log4j2.xml + ${project.parent.basedir}/beeline/src/main/resources/beeline-log4j2.properties conf - beeline-log4j2.xml.template + beeline-log4j2.properties.template ${project.parent.basedir}/llap-server/src/main/resources/llap-daemon-log4j.properties diff --git a/pom.xml b/pom.xml index c6df4a5..d937b63 100644 --- a/pom.xml +++ b/pom.xml @@ -981,7 +981,7 @@ ${maven.repo.local} local - ${test.log4j.scheme}${test.tmp.dir}/conf/hive-log4j2.xml + ${test.log4j.scheme}${test.tmp.dir}/conf/hive-log4j2.properties true ${test.tmp.dir} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java b/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java index 46662c4..a6b576f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java @@ -40,7 +40,7 @@ /** * A log4J2 Appender that simply counts logging events in four levels: - * fatal, error, warn and info. The class name is used in log4j2.xml + * fatal, error, warn and info. The class name can be used in log4j2.properties */ @Plugin(name = "HiveEventCounter", category = "Core", elementType = "appender", printObject = true) public class HiveEventCounter extends AbstractAppender { diff --git a/ql/src/main/resources/hive-exec-log4j.properties b/ql/src/main/resources/hive-exec-log4j.properties deleted file mode 100644 index 4c9c012..0000000 --- a/ql/src/main/resources/hive-exec-log4j.properties +++ /dev/null @@ -1,77 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Define some default values that can be overridden by system properties -hive.log.threshold=ALL -hive.root.logger=INFO,FA -hive.log.dir=${java.io.tmpdir}/${user.name} -hive.query.id=hadoop -hive.log.file=${hive.query.id}.log - -# Define the root logger to the system property "hadoop.root.logger". -log4j.rootLogger=${hive.root.logger}, EventCounter - -# Logging Threshold -log4j.threshhold=${hive.log.threshold} - -# -# File Appender -# - -log4j.appender.FA=org.apache.log4j.FileAppender -log4j.appender.FA.File=${hive.log.dir}/${hive.log.file} -log4j.appender.FA.layout=org.apache.log4j.PatternLayout - -# Pattern format: Date LogLevel LoggerName LogMessage -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -# Debugging Pattern format -log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p [%t(%x)]: %c{2} (%F:%M(%L)) - %m%n - - -# -# console -# Add "console" to rootlogger above if you want to use this -# - -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t(%x)] %p %c{2}: %m%n - -#custom logging levels -#log4j.logger.xxx=DEBUG - -# -# Event Counter Appender -# Sends counts of logging messages at different severity levels to Hadoop Metrics. -# -log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter - - -log4j.category.DataNucleus=ERROR,FA -log4j.category.Datastore=ERROR,FA -log4j.category.Datastore.Schema=ERROR,FA -log4j.category.JPOX.Datastore=ERROR,FA -log4j.category.JPOX.Plugin=ERROR,FA -log4j.category.JPOX.MetaData=ERROR,FA -log4j.category.JPOX.Query=ERROR,FA -log4j.category.JPOX.General=ERROR,FA -log4j.category.JPOX.Enhancer=ERROR,FA - - -# Silence useless ZK logs -log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,FA -log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,FA diff --git a/ql/src/main/resources/hive-exec-log4j2.properties b/ql/src/main/resources/hive-exec-log4j2.properties new file mode 100644 index 0000000..4fba04c --- /dev/null +++ b/ql/src/main/resources/hive-exec-log4j2.properties @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = HiveExecLog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.hive.log.level = INFO +property.hive.root.logger = FA +property.hive.query.id = hadoop +property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} +property.hive.log.file = ${sys:hive.query.id}.log + +# list of all appenders +appenders = console, FA + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + +# simple file appender +appender.FA.type = File +appender.FA.name = FA +appender.FA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file} +appender.FA.layout.type = PatternLayout +appender.FA.layout.pattern = %d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n + +# list of all loggers +loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX + +logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn +logger.NIOServerCnxn.level = WARN + +logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO +logger.ClientCnxnSocketNIO.level = WARN + +logger.DataNucleus.name = DataNucleus +logger.DataNucleus.level = ERROR + +logger.Datastore.name = Datastore +logger.Datastore.level = ERROR + +logger.JPOX.name = JPOX +logger.JPOX.level = ERROR + +# root logger +rootLogger.level = ${sys:hive.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} diff --git a/ql/src/main/resources/hive-exec-log4j2.xml b/ql/src/main/resources/hive-exec-log4j2.xml deleted file mode 100644 index d4728e4..0000000 --- a/ql/src/main/resources/hive-exec-log4j2.xml +++ /dev/null @@ -1,109 +0,0 @@ - - - - - - - INFO - FA - ${sys:java.io.tmpdir}/${sys:user.name} - hadoop - ${sys:hive.query.id}.log - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ql/src/main/resources/tez-container-log4j2.properties b/ql/src/main/resources/tez-container-log4j2.properties new file mode 100644 index 0000000..5d2b138 --- /dev/null +++ b/ql/src/main/resources/tez-container-log4j2.properties @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = TezContainerLog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.tez.container.log.level = INFO +property.tez.container.root.logger = CLA +property.tez.container.log.dir = ${sys:yarn.app.container.log.dir} +property.tez.container.log.file = syslog + +# list of all appenders +appenders = CLA + +# daily rolling file appender +appender.CLA.type = RollingFile +appender.CLA.name = CLA +appender.CLA.fileName = ${sys:tez.container.log.dir}/${sys:tez.container.log.file} +appender.CLA.filePattern = ${sys:tez.container.log.dir}/${sys:tez.container.log.file}.%d{yyyy-MM-dd} +appender.CLA.layout.type = PatternLayout +appender.CLA.layout.pattern = %d{ISO8601} %p [%t] %c{2}: %m%n +appender.CLA.policies.type = Policies +appender.CLA.policies.time.type = TimeBasedTriggeringPolicy +appender.CLA.policies.time.interval = 1 +appender.CLA.policies.time.modulate = true +appender.CLA.strategy.type = DefaultRolloverStrategy +appender.CLA.strategy.max = 30 + +# root logger +rootLogger.level = ${sys:tez.container.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:tez.container.root.logger} diff --git a/ql/src/main/resources/tez-container-log4j2.xml b/ql/src/main/resources/tez-container-log4j2.xml deleted file mode 100644 index 604e586..0000000 --- a/ql/src/main/resources/tez-container-log4j2.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - INFO - CLA - ${sys:yarn.app.container.log.dir} - syslog - - - - - - - - - - - - - - - - - - - - - diff --git a/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java b/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java index cc2ad4e..3cf8783 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java @@ -36,7 +36,7 @@ @Before public void setup() { - // programmatically set root logger level to INFO. By default if log4j2-test.xml is not + // programmatically set root logger level to INFO. By default if log4j2-test.properties is not // available root logger will use ERROR log level LoggerContext ctx = (LoggerContext) LogManager.getContext(false); Configuration config = ctx.getConfiguration(); @@ -47,7 +47,7 @@ public void setup() { @Test public void testStringAppender() throws Exception { - // Get the RootLogger which, if you don't have log4j2-test.xml defined, will only log ERRORs + // Get the RootLogger which, if you don't have log4j2-test.properties defined, will only log ERRORs Logger logger = LogManager.getRootLogger(); // Create a String Appender to capture log output StringAppender appender = StringAppender.createStringAppender("%m"); @@ -83,7 +83,7 @@ public void testHiveEventCounterAppender() throws Exception { logger.fatal("Test"); - // HiveEventCounter will be loaded from hive-log4j2-test.xml before tests are run. The 2 log + // HiveEventCounter will be loaded from hive-log4j2-test.properties before tests are run. The 2 log // info msgs from previous test case will also be counted along with 4 log info msgs in this // test and hence we assert for 6 here assertEquals(6, appender.getInfo()); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java index 9fd8516..b96de63 100755 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java @@ -249,7 +249,7 @@ public void testThriftTable() throws Throwable { * @throws Throwable */ public void testMetaStoreApiTiming() throws Throwable { - // Get the RootLogger which, if you don't have log4j2-test.xml defined, will only log ERRORs + // Get the RootLogger which, if you don't have log4j2-test.properties defined, will only log ERRORs Logger logger = LogManager.getLogger("hive.ql.metadata.Hive"); Level oldLevel = logger.getLevel(); LoggerContext ctx = (LoggerContext) LogManager.getContext(false); diff --git a/spark-client/src/test/resources/log4j2.properties b/spark-client/src/test/resources/log4j2.properties new file mode 100644 index 0000000..c65ef87 --- /dev/null +++ b/spark-client/src/test/resources/log4j2.properties @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = SparkClientLog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.spark.log.level = DEBUG +property.spark.root.logger = console + +# list of all appenders +appenders = console + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n + +# root logger +rootLogger.level = ${sys:spark.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:spark.root.logger} diff --git a/spark-client/src/test/resources/log4j2.xml b/spark-client/src/test/resources/log4j2.xml deleted file mode 100644 index a435069..0000000 --- a/spark-client/src/test/resources/log4j2.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - DEBUG - console - - - - - - - - - - - - - - - diff --git a/testutils/ptest2/pom.xml b/testutils/ptest2/pom.xml index 51a0aaf..8bf5242 100644 --- a/testutils/ptest2/pom.xml +++ b/testutils/ptest2/pom.xml @@ -26,7 +26,7 @@ limitations under the License. hive-ptest UTF-8 - 2.3 + 2.4 diff --git a/testutils/ptest2/src/main/resources/log4j2.properties b/testutils/ptest2/src/main/resources/log4j2.properties new file mode 100644 index 0000000..85f1b64 --- /dev/null +++ b/testutils/ptest2/src/main/resources/log4j2.properties @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = PTest2Log4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.hive.ptest.log.level = DEBUG +property.hive.ptest.root.logger = FILE +property.hive.ptest.log.dir = target +property.hive.ptest.log.file = ptest.log + +# list of all appenders +appenders = console, FILE + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +# simple file appender +appender.FILE.type = RollingFile +appender.FILE.name = FILE +appender.FILE.fileName = ${sys:hive.ptest.log.dir}/${sys:hive.ptest.log.file} +appender.FILE.filePattern = ${sys:hive.ptest.log.dir}/${sys:hive.ptest.log.file}.%i +appender.FILE.layout.type = PatternLayout +appender.FILE.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +appender.FILE.policies.type = Policies +appender.FILE.policies.size.type = SizeBasedTriggeringPolicy +appender.FILE.policies.size.size = 50MB +appender.FILE.strategy.type = DefaultRolloverStrategy +appender.FILE.strategy.max = 1 + +# list of all loggers +loggers = Http, SpringFramework, OrgJclouds, Jclouds, Hive, NIOServerCnxn, ClientCnxnSocketNIO + +logger.Http.name = org.apache.http +logger.Http.level = TRACE + +logger.SpringFramework.name = org.springframework +logger.SpringFramework.level = INFO + +logger.OrgJclouds.name = org.jclouds +logger.OrgJclouds.level = INFO + +logger.Jclouds.name = jclouds +logger.Jclouds.level = INFO + +logger.Hive.name = org.apache.hive +logger.Hive.level = DEBUG + +logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn +logger.NIOServerCnxn.level = WARN + +logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO +logger.ClientCnxnSocketNIO.level = WARN + +# root logger +rootLogger.level = ${sys:hive.ptest.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:hive.ptest.root.logger} diff --git a/testutils/ptest2/src/main/resources/log4j2.xml b/testutils/ptest2/src/main/resources/log4j2.xml deleted file mode 100644 index 8eb3234..0000000 --- a/testutils/ptest2/src/main/resources/log4j2.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - - - - - DEBUG - FILE - target - ptest.log - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -