diff --git pom.xml pom.xml index f36a259..96a07d2 100644 --- pom.xml +++ pom.xml @@ -121,7 +121,7 @@ 1.5.6 0.1 - 0.8.0 + 0.10.0 1.12.0 1.7.7 0.8.0.RELEASE diff --git ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java index 470f31c..a1a3327 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java @@ -198,7 +198,7 @@ private static Field toField(String name, TypeInfo typeInfo) { for (int i = 0; i < structSize; i++) { structFields.add(toField(fieldNames.get(i), fieldTypeInfos.get(i))); } - return new Field(name, FieldType.nullable(MinorType.MAP.getType()), structFields); + return new Field(name, FieldType.nullable(MinorType.STRUCT.getType()), structFields); case UNION: final UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; final List objectTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos(); diff --git ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java index 2961050..14eb4d8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java @@ -36,8 +36,8 @@ import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.complex.ListVector; -import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.NullableMapVector; +import org.apache.arrow.vector.complex.NonNullableStructVector; +import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -97,7 +97,7 @@ private final VectorAssignRow vectorAssignRow; private int batchSize; - private final NullableMapVector rootVector; + private final StructVector rootVector; Serializer(ArrowColumnarBatchSerDe serDe) throws SerDeException { MAX_BUFFERED_ROWS = HiveConf.getIntVar(serDe.conf, HIVE_ARROW_BATCH_SIZE); @@ -109,7 +109,7 @@ fieldSize = fieldTypeInfos.size(); // Init Arrow stuffs - rootVector = NullableMapVector.empty(null, serDe.rootAllocator); + rootVector = StructVector.empty(null, serDe.rootAllocator); // Init Hive stuffs vectorizedRowBatch = new VectorizedRowBatch(fieldSize); @@ -216,7 +216,7 @@ private void write(FieldVector arrowVector, ColumnVector hiveVector, TypeInfo ty writeList((ListVector) arrowVector, (ListColumnVector) hiveVector, (ListTypeInfo) typeInfo, size); break; case STRUCT: - writeStruct((MapVector) arrowVector, (StructColumnVector) hiveVector, (StructTypeInfo) typeInfo, size); + writeStruct((NonNullableStructVector) arrowVector, (StructColumnVector) hiveVector, (StructTypeInfo) typeInfo, size); break; case UNION: writeUnion(arrowVector, hiveVector, typeInfo, size); @@ -260,7 +260,7 @@ private void writeUnion(FieldVector arrowVector, ColumnVector hiveVector, TypeIn write(arrowVector, hiveObjectVector, objectTypeInfo, size); } - private void writeStruct(MapVector arrowVector, StructColumnVector hiveVector, + private void writeStruct(NonNullableStructVector arrowVector, StructColumnVector hiveVector, StructTypeInfo typeInfo, int size) { final List fieldNames = typeInfo.getAllStructFieldNames(); final List fieldTypeInfos = typeInfo.getAllStructFieldTypeInfos();