diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java index 2ad06fc128..38261821cf 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java @@ -1251,8 +1251,8 @@ private Object convertUnionRowColumn( final Object union = unionOI.create(); final int tag = deserializeRead.currentInt; - unionOI.addField(union, new StandardUnion((byte) tag, - convertComplexFieldRowColumn(unionColumnVector.fields[tag], batchIndex, fields[tag]))); + unionOI.setFieldAndTag(union, new StandardUnion((byte) tag, + convertComplexFieldRowColumn(unionColumnVector.fields[tag], batchIndex, fields[tag])), (byte) tag); deserializeRead.finishComplexVariableFieldsType(); return union; } diff --git ql/src/test/queries/clientpositive/orc_avro_partition_uniontype.q ql/src/test/queries/clientpositive/orc_avro_partition_uniontype.q new file mode 100644 index 0000000000..47ac526152 --- /dev/null +++ ql/src/test/queries/clientpositive/orc_avro_partition_uniontype.q @@ -0,0 +1,9 @@ +SET hive.exec.schema.evolution = false; + +CREATE TABLE avro_orc_partitioned_uniontype (a uniontype) PARTITIONED BY (b int) STORED AS ORC; + +INSERT INTO avro_orc_partitioned_uniontype PARTITION (b=1) SELECT create_union(1, true, value) FROM src LIMIT 5; + +ALTER TABLE avro_orc_partitioned_uniontype SET FILEFORMAT AVRO; + +SELECT * FROM avro_orc_partitioned_uniontype; diff --git ql/src/test/results/clientpositive/orc_avro_partition_uniontype.q.out ql/src/test/results/clientpositive/orc_avro_partition_uniontype.q.out new file mode 100644 index 0000000000..da8d9cc356 --- /dev/null +++ ql/src/test/results/clientpositive/orc_avro_partition_uniontype.q.out @@ -0,0 +1,40 @@ +PREHOOK: query: CREATE TABLE avro_orc_partitioned_uniontype (a uniontype) PARTITIONED BY (b int) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@avro_orc_partitioned_uniontype +POSTHOOK: query: CREATE TABLE avro_orc_partitioned_uniontype (a uniontype) PARTITIONED BY (b int) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@avro_orc_partitioned_uniontype +PREHOOK: query: INSERT INTO avro_orc_partitioned_uniontype PARTITION (b=1) SELECT create_union(1, true, value) FROM src LIMIT 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@avro_orc_partitioned_uniontype@b=1 +POSTHOOK: query: INSERT INTO avro_orc_partitioned_uniontype PARTITION (b=1) SELECT create_union(1, true, value) FROM src LIMIT 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@avro_orc_partitioned_uniontype@b=1 +POSTHOOK: Lineage: avro_orc_partitioned_uniontype PARTITION(b=1).a EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE avro_orc_partitioned_uniontype SET FILEFORMAT AVRO +PREHOOK: type: ALTERTABLE_FILEFORMAT +PREHOOK: Input: default@avro_orc_partitioned_uniontype +PREHOOK: Output: default@avro_orc_partitioned_uniontype +POSTHOOK: query: ALTER TABLE avro_orc_partitioned_uniontype SET FILEFORMAT AVRO +POSTHOOK: type: ALTERTABLE_FILEFORMAT +POSTHOOK: Input: default@avro_orc_partitioned_uniontype +POSTHOOK: Output: default@avro_orc_partitioned_uniontype +PREHOOK: query: SELECT * FROM avro_orc_partitioned_uniontype +PREHOOK: type: QUERY +PREHOOK: Input: default@avro_orc_partitioned_uniontype +PREHOOK: Input: default@avro_orc_partitioned_uniontype@b=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM avro_orc_partitioned_uniontype +POSTHOOK: type: QUERY +POSTHOOK: Input: default@avro_orc_partitioned_uniontype +POSTHOOK: Input: default@avro_orc_partitioned_uniontype@b=1 +#### A masked pattern was here #### +{1:"val_165"} 1 +{1:"val_27"} 1 +{1:"val_311"} 1 +{1:"val_86"} 1 +{1:"val_238"} 1 diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java index 7921de8d9c..76d458ba0f 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java @@ -471,7 +471,7 @@ public Object convert(Object input) { } Object inputFieldValue = inputOI.getField(input); - Object inputFieldTag = inputOI.getTag(input); + byte inputFieldTag = inputOI.getTag(input); Object outputFieldValue = null; int inputFieldTagIndex = ((Byte)inputFieldTag).intValue(); @@ -480,7 +480,7 @@ public Object convert(Object input) { outputFieldValue = fieldConverters.get(inputFieldTagIndex).convert(inputFieldValue); } - outputOI.addField(output, outputFieldValue); + outputOI.setFieldAndTag(output, outputFieldValue, inputFieldTag); return output; } diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java index 564d8d6045..e48ba03973 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java @@ -26,9 +26,9 @@ public abstract class SettableUnionObjectInspector implements UnionObjectInspector { - /* Create an empty object */ + /* Creates an empty union object. */ public abstract Object create(); - /* Add field to the object */ - public abstract Object addField(Object union, Object field); + /* Sets the field and tag in the union. Returns the union. */ + public abstract Object setFieldAndTag(Object union, Object field, byte tag); } diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java index 7b2868233f..d948965642 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java @@ -144,15 +144,19 @@ public String toString() { @Override public Object create() { - ArrayList a = new ArrayList(); - return a; + return new StandardUnion(); } @Override - public Object addField(Object union, Object field) { - ArrayList a = (ArrayList) union; - a.add(field); - return a; + public Object setFieldAndTag(Object union, Object field, byte tag) { + StandardUnion unionObject = (StandardUnion) union; + unionObject.setObject(field); + if (field == null) { + unionObject.setTag((byte) -1); + } else { + unionObject.setTag(tag); + } + return unionObject; } } diff --git serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java index 2e1bb22cea..219a6cd2ae 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java +++ serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java @@ -30,8 +30,8 @@ import org.apache.hadoop.hive.serde2.io.HiveCharWritable; import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; +import org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.StandardUnion; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; @@ -295,27 +295,30 @@ public void testObjectInspectorConverters() throws Throwable { Converter unionConverter0 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors), ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2)); - Object convertedObject0 = unionConverter0.convert(new StandardUnionObjectInspector.StandardUnion((byte)0, 1)); - List expectedObject0 = new ArrayList(); - expectedObject0.add("1"); + Object convertedObject0 = unionConverter0.convert(new StandardUnion((byte)0, 1)); + StandardUnion expectedObject0 = new StandardUnion(); + expectedObject0.setTag((byte) 0); + expectedObject0.setObject("1"); assertEquals(expectedObject0, convertedObject0); Converter unionConverter1 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors), ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2)); - Object convertedObject1 = unionConverter1.convert(new StandardUnionObjectInspector.StandardUnion((byte)1, "1")); - List expectedObject1 = new ArrayList(); - expectedObject1.add(1); + Object convertedObject1 = unionConverter1.convert(new StandardUnion((byte)1, "1")); + StandardUnion expectedObject1 = new StandardUnion(); + expectedObject1.setTag((byte) 1); + expectedObject1.setObject(1); assertEquals(expectedObject1, convertedObject1); Converter unionConverter2 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors), ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2)); - Object convertedObject2 = unionConverter2.convert(new StandardUnionObjectInspector.StandardUnion((byte)2, true)); - List expectedObject2 = new ArrayList(); - expectedObject2.add(true); + Object convertedObject2 = unionConverter2.convert(new StandardUnion((byte)2, true)); + StandardUnion expectedObject2 = new StandardUnion(); + expectedObject2.setTag((byte) 2); + expectedObject2.setObject(true); assertEquals(expectedObject2, convertedObject2); @@ -344,9 +347,10 @@ public void testObjectInspectorConverters() throws Throwable { Converter unionConverterExtra = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectorsExtra), ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectorsExtra2)); - Object convertedObjectExtra = unionConverterExtra.convert(new StandardUnionObjectInspector.StandardUnion((byte)2, true)); - List expectedObjectExtra = new ArrayList(); - expectedObjectExtra.add(null); + Object convertedObjectExtra = unionConverterExtra.convert(new StandardUnion((byte)2, true)); + StandardUnion expectedObjectExtra = new StandardUnion(); + expectedObjectExtra.setTag((byte) -1); + expectedObjectExtra.setObject(null); assertEquals(expectedObjectExtra, convertedObjectExtra); // we should get back null