From 27335008d5653e55a1b24f048b568e46417b8ebd Mon Sep 17 00:00:00 2001 From: ShubhamChaurasia Date: Wed, 17 Oct 2018 11:29:53 +0530 Subject: [PATCH] HIVE-20751: Upgrade arrow version to 0.10.0 --- pom.xml | 2 +- .../hadoop/hive/llap/LlapArrowRecordWriter.java | 4 ++-- .../hive/ql/io/arrow/ArrowColumnarBatchSerDe.java | 2 +- .../hive/ql/io/arrow/ArrowWrapperWritable.java | 8 ++++---- .../apache/hadoop/hive/ql/io/arrow/Serializer.java | 14 +++++++------- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pom.xml b/pom.xml index 75ebe9b4fa9..f440c8a1c84 100644 --- a/pom.xml +++ b/pom.xml @@ -123,7 +123,7 @@ 1.5.6 0.1 - 0.8.0 + 0.10.0 1.12.0 1.8.2 0.8.0.RELEASE diff --git a/ql/src/java/org/apache/hadoop/hive/llap/LlapArrowRecordWriter.java b/ql/src/java/org/apache/hadoop/hive/llap/LlapArrowRecordWriter.java index 9ee10480e77..4cd8a61c8f3 100644 --- a/ql/src/java/org/apache/hadoop/hive/llap/LlapArrowRecordWriter.java +++ b/ql/src/java/org/apache/hadoop/hive/llap/LlapArrowRecordWriter.java @@ -22,9 +22,9 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.complex.NonNullableStructVector; import org.apache.arrow.vector.ipc.ArrowStreamWriter; import org.apache.hadoop.hive.ql.io.arrow.ArrowWrapperWritable; -import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.RecordWriter; import org.apache.hadoop.mapred.Reporter; @@ -50,7 +50,7 @@ ArrowStreamWriter arrowStreamWriter; WritableByteChannelAdapter out; BufferAllocator allocator; - NullableMapVector rootVector; + NonNullableStructVector rootVector; public LlapArrowRecordWriter(WritableByteChannelAdapter out) { this.out = out; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java index ed82d2d01e9..04087073402 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java @@ -195,7 +195,7 @@ private static Field toField(String name, TypeInfo typeInfo) { for (int i = 0; i < structSize; i++) { structFields.add(toField(fieldNames.get(i), fieldTypeInfos.get(i))); } - return new Field(name, FieldType.nullable(MinorType.MAP.getType()), structFields); + return new Field(name, FieldType.nullable(MinorType.STRUCT.getType()), structFields); case UNION: final UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; final List objectTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowWrapperWritable.java b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowWrapperWritable.java index 40813fa12e3..53bee6b823f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowWrapperWritable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowWrapperWritable.java @@ -21,7 +21,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.hadoop.io.WritableComparable; -import org.apache.arrow.vector.complex.NullableMapVector; +import org.apache.arrow.vector.complex.NonNullableStructVector; import java.io.DataInput; import java.io.DataOutput; @@ -30,13 +30,13 @@ public class ArrowWrapperWritable implements WritableComparable { private VectorSchemaRoot vectorSchemaRoot; private BufferAllocator allocator; - private NullableMapVector rootVector; + private NonNullableStructVector rootVector; public ArrowWrapperWritable(VectorSchemaRoot vectorSchemaRoot) { this.vectorSchemaRoot = vectorSchemaRoot; } - public ArrowWrapperWritable(VectorSchemaRoot vectorSchemaRoot, BufferAllocator allocator, NullableMapVector rootVector) { + public ArrowWrapperWritable(VectorSchemaRoot vectorSchemaRoot, BufferAllocator allocator, NonNullableStructVector rootVector) { this.vectorSchemaRoot = vectorSchemaRoot; this.allocator = allocator; this.rootVector = rootVector; @@ -56,7 +56,7 @@ public BufferAllocator getAllocator() { return allocator; } - public NullableMapVector getRootVector() { + public NonNullableStructVector getRootVector() { return rootVector; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java index 7dffa6b02dc..c5079e11072 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java @@ -36,8 +36,8 @@ import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.complex.ListVector; -import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.NullableMapVector; +import org.apache.arrow.vector.complex.NonNullableStructVector; +import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.holders.DecimalHolder; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.Types; @@ -110,7 +110,7 @@ private List fieldNames; private int fieldSize; - private final NullableMapVector rootVector; + private final StructVector rootVector; private final DecimalHolder decimalHolder = new DecimalHolder(); //Constructor for non-serde serialization @@ -125,7 +125,7 @@ public Serializer(Configuration conf, String attemptId, List typeInfos attemptId, childAllocatorReservation, childAllocatorLimit); - rootVector = NullableMapVector.empty(null, allocator); + rootVector = StructVector.empty(null, allocator); //These last fields are unused in non-serde usage vectorizedRowBatch = null; vectorAssignRow = null; @@ -151,7 +151,7 @@ public Serializer(Configuration conf, String attemptId, List typeInfos fieldNames = structTypeInfo.getAllStructFieldNames(); fieldSize = fieldTypeInfos.size(); // Init Arrow stuffs - rootVector = NullableMapVector.empty(null, allocator); + rootVector = StructVector.empty(null, allocator); // Init Hive stuffs vectorizedRowBatch = new VectorizedRowBatch(fieldSize); @@ -292,7 +292,7 @@ private void write(FieldVector arrowVector, ColumnVector hiveVector, TypeInfo ty writeList((ListVector) arrowVector, (ListColumnVector) hiveVector, (ListTypeInfo) typeInfo, size, vectorizedRowBatch, isNative); break; case STRUCT: - writeStruct((MapVector) arrowVector, (StructColumnVector) hiveVector, (StructTypeInfo) typeInfo, size, vectorizedRowBatch, isNative); + writeStruct((NonNullableStructVector) arrowVector, (StructColumnVector) hiveVector, (StructTypeInfo) typeInfo, size, vectorizedRowBatch, isNative); break; case UNION: writeUnion(arrowVector, hiveVector, typeInfo, size, vectorizedRowBatch, isNative); @@ -336,7 +336,7 @@ private void writeUnion(FieldVector arrowVector, ColumnVector hiveVector, TypeIn write(arrowVector, hiveObjectVector, objectTypeInfo, size, vectorizedRowBatch, isNative); } - private void writeStruct(MapVector arrowVector, StructColumnVector hiveVector, + private void writeStruct(NonNullableStructVector arrowVector, StructColumnVector hiveVector, StructTypeInfo typeInfo, int size, VectorizedRowBatch vectorizedRowBatch, boolean isNative) { final List fieldNames = typeInfo.getAllStructFieldNames(); final List fieldTypeInfos = typeInfo.getAllStructFieldTypeInfos();