diff --git a/itests/qtest-accumulo/pom.xml b/itests/qtest-accumulo/pom.xml index 7403a15..f7325dc 100644 --- a/itests/qtest-accumulo/pom.xml +++ b/itests/qtest-accumulo/pom.xml @@ -123,7 +123,7 @@ - com.esotericsoftware.kryo + com.esotericsoftware kryo ${kryo.version} test diff --git a/pom.xml b/pom.xml index 4a90cef..e46fe40 100644 --- a/pom.xml +++ b/pom.xml @@ -144,7 +144,7 @@ 3.5.2 20090211 4.11 - 2.22 + 3.0.3 0.9.3 0.9.3 2.4 @@ -238,8 +238,8 @@ - com.esotericsoftware.kryo - kryo + com.esotericsoftware + kryo-shaded ${kryo.version} diff --git a/ql/pom.xml b/ql/pom.xml index 9420a62..d893099 100644 --- a/ql/pom.xml +++ b/ql/pom.xml @@ -72,8 +72,8 @@ - com.esotericsoftware.kryo - kryo + com.esotericsoftware + kryo-shaded ${kryo.version} @@ -594,16 +594,20 @@ spark-core_${scala.binary.version} ${spark.version} true - - - org.slf4j - slf4j-log4j12 - - - commmons-logging - commons-logging - - + + + com.esotericsoftware.kryo + kryo + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + com.sun.jersey @@ -746,7 +750,9 @@ org.apache.hive:hive-serde org.apache.hive:hive-llap-client org.apache.hive:hive-metastore - com.esotericsoftware.kryo:kryo + com.esotericsoftware:kryo-shaded + com.esotericsoftware:minlog + org.objenesis:objenesis org.apache.parquet:parquet-hadoop-bundle org.apache.thrift:libthrift org.apache.thrift:libfb303 @@ -779,6 +785,10 @@ com.esotericsoftware org.apache.hive.com.esotericsoftware + + org.objenesis + org.apache.hive.org.objenesis + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index de2eb98..5e9af3c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -75,7 +75,6 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.Deflater; @@ -195,13 +194,12 @@ import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.Shell; import org.apache.hive.common.util.ReflectionUtil; -import org.slf4j.Logger; +import org.objenesis.strategy.StdInstantiatorStrategy; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import com.esotericsoftware.kryo.serializers.FieldSerializer; -import com.esotericsoftware.shaded.org.objenesis.strategy.StdInstantiatorStrategy; import com.google.common.base.Preconditions; /** @@ -1099,10 +1097,16 @@ protected Kryo initialValue() { kryo.register(java.sql.Date.class, new SqlDateSerializer()); kryo.register(java.sql.Timestamp.class, new TimestampSerializer()); kryo.register(Path.class, new PathSerializer()); - kryo.setInstantiatorStrategy(new StdInstantiatorStrategy()); + ((Kryo.DefaultInstantiatorStrategy) kryo.getInstantiatorStrategy()).setFallbackInstantiatorStrategy( + new StdInstantiatorStrategy()); removeField(kryo, Operator.class, "colExprMap"); - removeField(kryo, ColumnInfo.class, "objectInspector"); removeField(kryo, AbstractOperatorDesc.class, "statistics"); + kryo.register(MapWork.class); + kryo.register(ReduceWork.class); + kryo.register(TableDesc.class); + kryo.register(UnionOperator.class); + kryo.register(FileSinkOperator.class); + kryo.register(HiveIgnoreKeyTextOutputFormat.class); return kryo; }; }; @@ -1121,15 +1125,19 @@ protected synchronized Kryo initialValue() { kryo.register(java.sql.Date.class, new SqlDateSerializer()); kryo.register(java.sql.Timestamp.class, new TimestampSerializer()); kryo.register(Path.class, new PathSerializer()); - kryo.setInstantiatorStrategy(new StdInstantiatorStrategy()); + ((Kryo.DefaultInstantiatorStrategy) kryo.getInstantiatorStrategy()).setFallbackInstantiatorStrategy(new StdInstantiatorStrategy()); removeField(kryo, Operator.class, "colExprMap"); removeField(kryo, ColumnInfo.class, "objectInspector"); + removeField(kryo, AbstractOperatorDesc.class, "statistics"); kryo.register(SparkEdgeProperty.class); kryo.register(MapWork.class); kryo.register(ReduceWork.class); kryo.register(SparkWork.class); kryo.register(TableDesc.class); kryo.register(Pair.class); + kryo.register(UnionOperator.class); + kryo.register(FileSinkOperator.class); + kryo.register(HiveIgnoreKeyTextOutputFormat.class); return kryo; }; }; @@ -1143,7 +1151,16 @@ protected Kryo initialValue() { kryo.register(java.sql.Date.class, new SqlDateSerializer()); kryo.register(java.sql.Timestamp.class, new TimestampSerializer()); kryo.register(Path.class, new PathSerializer()); - kryo.setInstantiatorStrategy(new StdInstantiatorStrategy()); + ((Kryo.DefaultInstantiatorStrategy) kryo.getInstantiatorStrategy()).setFallbackInstantiatorStrategy( + new StdInstantiatorStrategy()); + removeField(kryo, Operator.class, "colExprMap"); + removeField(kryo, AbstractOperatorDesc.class, "statistics"); + kryo.register(MapWork.class); + kryo.register(ReduceWork.class); + kryo.register(TableDesc.class); + kryo.register(UnionOperator.class); + kryo.register(FileSinkOperator.class); + kryo.register(HiveIgnoreKeyTextOutputFormat.class); return kryo; }; }; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java index f4e5873..73e8f6d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java @@ -31,8 +31,6 @@ import java.util.Map.Entry; import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; @@ -64,8 +62,6 @@ @SuppressWarnings({"serial", "deprecation"}) public class MapWork extends BaseWork { - private static final Logger LOG = LoggerFactory.getLogger(MapWork.class); - // use LinkedHashMap to make sure the iteration order is // deterministic, to ease testing private LinkedHashMap> pathToAliases = new LinkedHashMap>(); @@ -548,17 +544,6 @@ public void configureJobConf(JobConf job) { } } - public void logPathToAliases() { - if (LOG.isDebugEnabled()) { - LOG.debug("LOGGING PATH TO ALIASES"); - for (Map.Entry> entry: pathToAliases.entrySet()) { - for (String a: entry.getValue()) { - LOG.debug("Path: " + entry.getKey() + ", Alias: " + a); - } - } - } - } - public void setDummyTableScan(boolean dummyTableScan) { this.dummyTableScan = dummyTableScan; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java index 8211346..0ac625f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java @@ -23,11 +23,8 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorUtils; @@ -60,8 +57,6 @@ public ReduceWork(String name) { super(name); } - private static transient final Logger LOG = LoggerFactory.getLogger(ReduceWork.class); - // schema of the map-reduce 'key' object - this is homogeneous private TableDesc keyDesc; diff --git a/spark-client/pom.xml b/spark-client/pom.xml index a0bbe56..9d2b418 100644 --- a/spark-client/pom.xml +++ b/spark-client/pom.xml @@ -39,8 +39,8 @@ - com.esotericsoftware.kryo - kryo + com.esotericsoftware + kryo-shaded com.google.guava @@ -59,16 +59,20 @@ org.apache.spark spark-core_${scala.binary.version} ${spark.version} - - - org.slf4j - slf4j-log4j12 - - - commmons-logging - commons-logging - - + + + com.esotericsoftware.kryo + kryo + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + junit diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/rpc/KryoMessageCodec.java b/spark-client/src/main/java/org/apache/hive/spark/client/rpc/KryoMessageCodec.java index 197f113..9e789cf 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/rpc/KryoMessageCodec.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/rpc/KryoMessageCodec.java @@ -23,19 +23,20 @@ import java.util.Arrays; import java.util.List; +import org.objenesis.strategy.StdInstantiatorStrategy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.ByteBufferInputStream; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; -import com.esotericsoftware.shaded.org.objenesis.strategy.StdInstantiatorStrategy; import com.google.common.base.Preconditions; + import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.ByteToMessageCodec; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * Codec that serializes / deserializes objects using Kryo. Objects are encoded with a 4-byte * header with the length of the serialized data. @@ -59,7 +60,7 @@ protected Kryo initialValue() { kryo.register(klass, REG_ID_BASE + count); count++; } - kryo.setInstantiatorStrategy(new StdInstantiatorStrategy()); + kryo.setInstantiatorStrategy(new Kryo.DefaultInstantiatorStrategy(new StdInstantiatorStrategy())); return kryo; } };