From 9b49dfc80cd1111bc1ffa124723c24466e4c9adc Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Wed, 28 Oct 2015 17:23:16 -0700 Subject: [PATCH] HIVE-12289 : Make sure slf4j-log4j12 jar is not in classpath --- common/pom.xml | 24 ++++++++++- .../org/apache/hadoop/hive/common/JavaUtils.java | 11 +++-- .../hive/metastore/hbase/TestHBaseSchemaTool.java | 6 +-- .../hive/metastore/hbase/TestHBaseSchemaTool2.java | 4 +- llap-client/pom.xml | 24 ++++++++++- llap-server/pom.xml | 48 ++++++++++++++++++++-- metastore/pom.xml | 32 +++++++++++++-- .../hive/metastore/hbase/HBaseSchemaTool.java | 4 -- pom.xml | 46 +++++++++++++++++++-- ql/pom.xml | 22 +++++++++- .../hive/ql/exec/mr/HadoopJobExecHelper.java | 11 +++-- serde/pom.xml | 24 ++++++++++- service/pom.xml | 12 +++++- shims/0.23/pom.xml | 36 ++++++++++++++-- storage-api/pom.xml | 12 +++++- 15 files changed, 270 insertions(+), 46 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index f9c5629..2292fdf 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -90,13 +90,33 @@ hadoop-common ${hadoop.version} true - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.hadoop hadoop-mapreduce-client-core ${hadoop.version} true - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + com.google.code.tempus-fugit diff --git a/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java b/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java index dc3a4ae..87bc99e 100644 --- a/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java @@ -27,8 +27,8 @@ import java.net.URLClassLoader; import java.util.Arrays; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Collection of Java class loading/reflection related utilities common across @@ -36,7 +36,7 @@ */ public final class JavaUtils { - private static final Log LOG = LogFactory.getLog(JavaUtils.class); + private static final Logger LOG = LoggerFactory.getLogger(JavaUtils.class); private static final Method SUN_MISC_UTIL_RELEASE; static { @@ -56,10 +56,10 @@ /** * Standard way of getting classloader in Hive code (outside of Hadoop). - * + * * Uses the context loader to get access to classpaths to auxiliary and jars * added with 'add jar' command. Falls back to current classloader. - * + * * In Hadoop-related code, we use Configuration.getClassLoader(). */ public static ClassLoader getClassLoader() { @@ -133,7 +133,6 @@ public static void closeClassLoader(ClassLoader loader) throws IOException { newOutputStream.close(); } } - LogFactory.release(loader); } /** diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java index d3b8615..24bebb8 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java @@ -19,8 +19,6 @@ package org.apache.hadoop.hive.metastore.hbase; import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; @@ -56,8 +54,8 @@ import java.util.List; public class TestHBaseSchemaTool extends HBaseIntegrationTests { - private static final Log LOG = LogFactory.getLog(TestHBaseSchemaTool.class.getName()); - private String lsep = System.getProperty("line.separator"); + + private final String lsep = System.getProperty("line.separator"); @BeforeClass public static void startup() throws Exception { diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool2.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool2.java index e343035..0c95b2f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool2.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool2.java @@ -18,8 +18,6 @@ package org.apache.hadoop.hive.metastore.hbase; import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -32,7 +30,7 @@ * This is in a separate class because install tests shouldn't set up the metastore first. */ public class TestHBaseSchemaTool2 extends HBaseIntegrationTests { - private static final Log LOG = LogFactory.getLog(TestHBaseSchemaTool.class.getName()); + private String lsep = System.getProperty("line.separator"); @BeforeClass diff --git a/llap-client/pom.xml b/llap-client/pom.xml index 02243f8..a53ad58 100644 --- a/llap-client/pom.xml +++ b/llap-client/pom.xml @@ -65,7 +65,17 @@ hadoop-common ${hadoop.version} true - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.hadoop hadoop-mapreduce-client-core @@ -97,7 +107,17 @@ ${hadoop.version} tests test - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.hadoop hadoop-hdfs diff --git a/llap-server/pom.xml b/llap-server/pom.xml index 4be45a5..d019414 100644 --- a/llap-server/pom.xml +++ b/llap-server/pom.xml @@ -90,13 +90,33 @@ hadoop-common ${hadoop.version} true - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.hadoop hadoop-mapreduce-client-core ${hadoop.version} true - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.hadoop hadoop-yarn-registry @@ -108,7 +128,17 @@ tez-runtime-internals ${tez.version} true - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.tez tez-runtime-library @@ -134,7 +164,17 @@ ${hadoop.version} tests test - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.hadoop hadoop-hdfs diff --git a/metastore/pom.xml b/metastore/pom.xml index e3f825b..195e8d2 100644 --- a/metastore/pom.xml +++ b/metastore/pom.xml @@ -59,7 +59,13 @@ org.apache.hbase hbase-client ${hbase.version} - + + + commmons-logging + commons-logging + + + com.jolbox bonecp @@ -121,13 +127,33 @@ hadoop-common ${hadoop.version} true - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.hadoop hadoop-mapreduce-client-core ${hadoop.version} true - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.thrift libfb303 diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java index 567970c..b4f8734 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java @@ -26,8 +26,6 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.codec.binary.Base64; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import java.io.PrintStream; @@ -40,8 +38,6 @@ */ public class HBaseSchemaTool { - static final private Log LOG = LogFactory.getLog(HBaseReadWrite.class.getName()); - public static void main(String[] args) { Options options = new Options(); diff --git a/pom.xml b/pom.xml index 282d077..81ed963 100644 --- a/pom.xml +++ b/pom.xml @@ -495,7 +495,23 @@ zookeeper ${zookeeper.version} + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.httpcomponents + httpcore + + org.apache.httpcomponents + httpclient + + org.jboss.netty netty @@ -602,7 +618,11 @@ hadoop-common ${hadoop.version} - + + org.slf4j + slf4j-log4j12 + + commmons-logging commons-logging @@ -625,12 +645,32 @@ org.apache.hadoop hadoop-mapreduce-client-jobclient ${hadoop.version} - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.hadoop hadoop-mapreduce-client-core ${hadoop.version} - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.hadoop hadoop-minikdc diff --git a/ql/pom.xml b/ql/pom.xml index 005c232..b073597 100644 --- a/ql/pom.xml +++ b/ql/pom.xml @@ -90,7 +90,17 @@ commons-httpclient commons-httpclient ${commons-httpclient.version} - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + commons-io commons-io @@ -168,7 +178,15 @@ hadoop-common ${hadoop.version} - + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + javax.servlet servlet-api diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java index 44dfe3e..5f35630 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java @@ -30,8 +30,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.MapRedStats; @@ -61,10 +59,11 @@ import org.apache.logging.log4j.core.Logger; import org.apache.logging.log4j.core.appender.FileAppender; import org.apache.logging.log4j.core.appender.RollingFileAppender; +import org.slf4j.LoggerFactory; public class HadoopJobExecHelper { - static final private Log LOG = LogFactory.getLog(HadoopJobExecHelper.class.getName()); + static final private org.slf4j.Logger LOG = LoggerFactory.getLogger(HadoopJobExecHelper.class.getName()); protected transient JobConf job; protected Task task; @@ -76,8 +75,8 @@ protected transient int lastReduceProgress; public transient JobID jobId; - private LogHelper console; - private HadoopJobExecHook callBackObj; + private final LogHelper console; + private final HadoopJobExecHook callBackObj; /** * Update counters relevant to this task. @@ -186,7 +185,7 @@ public static void killRunningJobs() { System.err.println("killing job with: " + rj.getID()); rj.killJob(); } catch (Exception e) { - LOG.warn(e); + LOG.warn("Failed to kill job", e); System.err.println("Failed to kill job: "+ rj.getID()); // do nothing } diff --git a/serde/pom.xml b/serde/pom.xml index 772ce71..4e854c2 100644 --- a/serde/pom.xml +++ b/serde/pom.xml @@ -85,7 +85,17 @@ hadoop-common ${hadoop.version} true - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.hadoop hadoop-mapreduce-client-core @@ -124,7 +134,17 @@ ${hadoop.version} tests test - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.hadoop hadoop-hdfs diff --git a/service/pom.xml b/service/pom.xml index 7095448..22234d9 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -59,7 +59,17 @@ net.sf.jpam jpam ${jpam.version} - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + commons-lang diff --git a/shims/0.23/pom.xml b/shims/0.23/pom.xml index 17efde8..72d49e2 100644 --- a/shims/0.23/pom.xml +++ b/shims/0.23/pom.xml @@ -51,7 +51,17 @@ hadoop-common ${hadoop.version} true - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.hadoop hadoop-hdfs @@ -70,14 +80,34 @@ hadoop-mapreduce-client-core ${hadoop.version} true - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.hadoop hadoop-mapreduce-client-jobclient ${hadoop.version} test-jar true - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + org.apache.tez tez-api diff --git a/storage-api/pom.xml b/storage-api/pom.xml index 0af0d27..ae9da60 100644 --- a/storage-api/pom.xml +++ b/storage-api/pom.xml @@ -38,7 +38,17 @@ hadoop-common ${hadoop.version} true - + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + junit junit -- 1.7.12.4 (Apple Git-37)