diff --git kryo-registrator/pom.xml kryo-registrator/pom.xml index ea2eb6508c..a9c276362a 100644 --- kryo-registrator/pom.xml +++ kryo-registrator/pom.xml @@ -38,6 +38,11 @@ ${project.version} true + + com.esotericsoftware + kryo + ${kryo.version} + org.apache.spark spark-core_${scala.binary.version} diff --git kryo-registrator/src/main/java/org/apache/hive/spark/HiveKryoRegistrator.java kryo-registrator/src/main/java/org/apache/hive/spark/HiveKryoRegistrator.java index 001ab8e086..54307ca159 100644 --- kryo-registrator/src/main/java/org/apache/hive/spark/HiveKryoRegistrator.java +++ kryo-registrator/src/main/java/org/apache/hive/spark/HiveKryoRegistrator.java @@ -46,7 +46,7 @@ public void write(Kryo kryo, Output output, HiveKey object) { output.writeVarInt(object.hashCode(), false); } - public HiveKey read(Kryo kryo, Input input, Class type) { + public HiveKey read(Kryo kryo, Input input, Class type) { int len = input.readVarInt(true); byte[] bytes = new byte[len]; input.readBytes(bytes); @@ -61,7 +61,7 @@ public void write(Kryo kryo, Output output, BytesWritable object) { output.write(object.getBytes(), 0, object.getLength()); } - public BytesWritable read(Kryo kryo, Input input, Class type) { + public BytesWritable read(Kryo kryo, Input input, Class type) { int len = input.readVarInt(true); byte[] bytes = new byte[len]; input.readBytes(bytes); diff --git kryo-registrator/src/main/java/org/apache/hive/spark/NoHashCodeKryoSerializer.java kryo-registrator/src/main/java/org/apache/hive/spark/NoHashCodeKryoSerializer.java index d4bcc5bdc8..a994a5bc83 100644 --- kryo-registrator/src/main/java/org/apache/hive/spark/NoHashCodeKryoSerializer.java +++ kryo-registrator/src/main/java/org/apache/hive/spark/NoHashCodeKryoSerializer.java @@ -55,7 +55,7 @@ public void write(Kryo kryo, Output output, HiveKey object) { output.write(object.getBytes(), 0, object.getLength()); } - public HiveKey read(Kryo kryo, Input input, Class type) { + public HiveKey read(Kryo kryo, Input input, Class type) { int len = input.readVarInt(true); byte[] bytes = new byte[len]; input.readBytes(bytes); diff --git pom.xml pom.xml index a005f35632..beaa7f6b1d 100644 --- pom.xml +++ pom.xml @@ -184,7 +184,7 @@ 3.5.2 1.8 4.11 - 4.0.2 + 5.0.0-RC4 1.10.0 0.9.3 @@ -256,7 +256,7 @@ com.esotericsoftware - kryo-shaded + kryo ${kryo.version} diff --git ql/pom.xml ql/pom.xml index 8b0c02b9b1..7f400f872c 100644 --- ql/pom.xml +++ ql/pom.xml @@ -101,7 +101,7 @@ com.esotericsoftware - kryo-shaded + kryo ${kryo.version} @@ -949,7 +949,7 @@ org.apache.hive:hive-standalone-metastore-common org.apache.hive:hive-standalone-metastore-server org.apache.hive:hive-service-rpc - com.esotericsoftware:kryo-shaded + com.esotericsoftware:kryo com.esotericsoftware:minlog org.objenesis:objenesis org.apache.parquet:parquet-hadoop-bundle diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java index f19f394faf..61438c2e1e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java @@ -68,6 +68,7 @@ import org.slf4j.LoggerFactory; import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.util.DefaultInstantiatorStrategy; import com.esotericsoftware.kryo.Registration; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; @@ -236,7 +237,7 @@ public Kryo create() { kryo.register(CopyOnFirstWriteProperties.class, new CopyOnFirstWritePropertiesSerializer()); kryo.register(PartitionDesc.class, new PartitionDescSerializer(kryo, PartitionDesc.class)); - ((Kryo.DefaultInstantiatorStrategy) kryo.getInstantiatorStrategy()) + ((DefaultInstantiatorStrategy) kryo.getInstantiatorStrategy()) .setFallbackInstantiatorStrategy( new StdInstantiatorStrategy()); removeField(kryo, AbstractOperatorDesc.class, "colExprMap"); @@ -303,7 +304,7 @@ private static void removeField(Kryo kryo, Class type, String fieldName) { com.esotericsoftware.kryo.Serializer { @Override - public Timestamp read(Kryo kryo, Input input, Class clazz) { + public Timestamp read(Kryo kryo, Input input, Class clazz) { Timestamp ts = new Timestamp(input.readLong()); ts.setNanos(input.readInt()); return ts; @@ -326,7 +327,7 @@ public void write(Kryo kryo, Output output, TimestampTZ object) { } @Override - public TimestampTZ read(Kryo kryo, Input input, Class type) { + public TimestampTZ read(Kryo kryo, Input input, Class type) { long seconds = input.readLong(); int nanos = input.readInt(); String zoneId = input.readString(); @@ -342,7 +343,7 @@ public TimestampTZ read(Kryo kryo, Input input, Class type) { com.esotericsoftware.kryo.Serializer { @Override - public java.sql.Date read(Kryo kryo, Input input, Class clazz) { + public java.sql.Date read(Kryo kryo, Input input, Class clazz) { return new java.sql.Date(input.readLong()); } @@ -360,7 +361,7 @@ public void write(Kryo kryo, Output output, Path path) { } @Override - public Path read(Kryo kryo, Input input, Class type) { + public Path read(Kryo kryo, Input input, Class type) { return new Path(URI.create(input.readString())); } } @@ -409,7 +410,7 @@ private static Field getOffsetField(Class clazz) throws NoSuchFieldException } @Override - public List read(final Kryo kryo, final Input input, final Class> clazz) { + public List read(final Kryo kryo, final Input input, final Class> clazz) { kryo.reference(FAKE_REFERENCE); final List list = (List) kryo.readClassAndObject(input); final int fromIndex = input.readInt(true); @@ -476,7 +477,7 @@ public ArraysAsListSerializer() { } @Override - public List read(final Kryo kryo, final Input input, final Class> type) { + public List read(final Kryo kryo, final Input input, final Class> type) { final int length = input.readInt(true); Class componentType = kryo.readClass(input).getType(); if (componentType.isPrimitive()) { @@ -544,7 +545,7 @@ public void write(final Kryo kryo, final Output output, final List obj) { * superclass declares most of its fields transient. */ private static class CopyOnFirstWritePropertiesSerializer extends - com.esotericsoftware.kryo.serializers.MapSerializer { + com.esotericsoftware.kryo.serializers.MapSerializer { @Override public void write(Kryo kryo, Output output, Map map) { @@ -555,7 +556,7 @@ public void write(Kryo kryo, Output output, Map map) { } @Override - public Map read(Kryo kryo, Input input, Class type) { + public Map read(Kryo kryo, Input input, Class type) { Map map = super.read(kryo, input, type); Properties ip = kryo.readObjectOrNull(input, Properties.class); ((CopyOnFirstWriteProperties) map).setInterned(ip); @@ -576,7 +577,7 @@ public Map read(Kryo kryo, Input input, Class type) { } @Override - public PartitionDesc read(Kryo kryo, Input input, Class type) { + public PartitionDesc read(Kryo kryo, Input input, Class type) { PartitionDesc partitionDesc = super.read(kryo, input, type); // The set methods in PartitionDesc intern the any duplicate strings which is why we call them // during de-serialization diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/KeyValueContainer.java ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/KeyValueContainer.java index 6d06bff5ab..4861ed8717 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/KeyValueContainer.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/KeyValueContainer.java @@ -191,7 +191,7 @@ public boolean hasNext() { pair.setRight(readValue(input)); } - if (input.eof()) { + if (input.end()) { input.close(); input = null; } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java index 60fc47d59b..1a848a1d2e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java @@ -174,7 +174,7 @@ public ROW next() { SerializationUtilities.releaseKryo(kryo); } - if (input.eof()) { + if (input.end()) { input.close(); input = null; } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveKVResultCache.java ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveKVResultCache.java index 253e310b77..ee78bf77e3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveKVResultCache.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveKVResultCache.java @@ -228,7 +228,7 @@ public synchronized boolean hasNext() { pair.setLeft(readHiveKey(input)); pair.setRight(readValue(input)); } - if (input.eof()) { + if (input.end()) { input.close(); input = null; } diff --git spark-client/pom.xml spark-client/pom.xml index a3cf922528..a40f47b9a6 100644 --- spark-client/pom.xml +++ spark-client/pom.xml @@ -39,7 +39,7 @@ com.esotericsoftware - kryo-shaded + kryo com.google.guava diff --git spark-client/src/main/java/org/apache/hive/spark/client/rpc/KryoMessageCodec.java spark-client/src/main/java/org/apache/hive/spark/client/rpc/KryoMessageCodec.java index d3a68125c7..5247d6991c 100644 --- spark-client/src/main/java/org/apache/hive/spark/client/rpc/KryoMessageCodec.java +++ spark-client/src/main/java/org/apache/hive/spark/client/rpc/KryoMessageCodec.java @@ -35,6 +35,7 @@ import com.esotericsoftware.kryo.io.ByteBufferInputStream; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; +import com.esotericsoftware.kryo.util.DefaultInstantiatorStrategy; import com.google.common.base.Preconditions; @@ -66,7 +67,7 @@ protected Kryo initialValue() { count++; } kryo.register(BaseProtocol.JobResult.class, new JobResultSerializer(), count); - kryo.setInstantiatorStrategy(new Kryo.DefaultInstantiatorStrategy(new StdInstantiatorStrategy())); + kryo.setInstantiatorStrategy(new DefaultInstantiatorStrategy(new StdInstantiatorStrategy())); return kryo; } };