diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java index 8ef8ce1..24b3d4e 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java @@ -424,8 +424,9 @@ public Object convert(Object input) { UnionObjectInspector inputOI; SettableUnionObjectInspector outputOI; - List inputFields; - List outputFields; + // Object inspectors for the tags for the input and output unionss + List inputTagsOIs; + List outputTagsOIs; ArrayList fieldConverters; @@ -436,14 +437,14 @@ public UnionConverter(ObjectInspector inputOI, if (inputOI instanceof UnionObjectInspector) { this.inputOI = (UnionObjectInspector)inputOI; this.outputOI = outputOI; - inputFields = this.inputOI.getObjectInspectors(); - outputFields = outputOI.getObjectInspectors(); + inputTagsOIs = this.inputOI.getObjectInspectors(); + outputTagsOIs = outputOI.getObjectInspectors(); // If the output has some extra fields, set them to NULL in convert(). - int minFields = Math.min(inputFields.size(), outputFields.size()); + int minFields = Math.min(inputTagsOIs.size(), outputTagsOIs.size()); fieldConverters = new ArrayList(minFields); for (int f = 0; f < minFields; f++) { - fieldConverters.add(getConverter(inputFields.get(f), outputFields.get(f))); + fieldConverters.add(getConverter(inputTagsOIs.get(f), outputTagsOIs.get(f))); } // Create an empty output object which will be populated when convert() is invoked. @@ -461,18 +462,18 @@ public Object convert(Object input) { return null; } - int minFields = Math.min(inputFields.size(), outputFields.size()); - // Convert the fields - for (int f = 0; f < minFields; f++) { - Object outputFieldValue = fieldConverters.get(f).convert(inputOI); - outputOI.addField(output, (ObjectInspector)outputFieldValue); - } + Object inputFieldValue = inputOI.getField(input); + Object inputFieldTag = inputOI.getTag(input); + Object outputFieldValue = null; - // set the extra fields to null - for (int f = minFields; f < outputFields.size(); f++) { - outputOI.addField(output, null); + int inputFieldTagIndex = ((Byte)inputFieldTag).intValue(); + + if (inputFieldTagIndex >= 0 && inputFieldTagIndex < fieldConverters.size()) { + outputFieldValue = fieldConverters.get(inputFieldTagIndex).convert(inputFieldValue); } + outputOI.addField(output, outputFieldValue); + return output; } } diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java index a64aee0..564d8d6 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java @@ -29,6 +29,6 @@ /* Create an empty object */ public abstract Object create(); - /* Add fields to the object */ - public abstract Object addField(Object union, ObjectInspector oi); + /* Add field to the object */ + public abstract Object addField(Object union, Object field); } diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java index d1b11e8..f26c9ec 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java @@ -124,9 +124,9 @@ public Object create() { } @Override - public Object addField(Object union, ObjectInspector oi) { + public Object addField(Object union, Object field) { ArrayList a = (ArrayList) union; - a.add(oi); + a.add(field); return a; } diff --git serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java index 1185283..dd18517 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java +++ serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hive.serde2.objectinspector; +import java.util.ArrayList; +import java.util.List; + import junit.framework.TestCase; import org.apache.hadoop.hive.common.type.HiveDecimal; @@ -168,6 +171,90 @@ public void testObjectInspectorConverters() throws Throwable { {(byte)'h', (byte)'i',(byte)'v',(byte)'e'}), baConverter.convert(new Text("hive"))); assertEquals("BAConverter", null, baConverter.convert(null)); + + // Union + ArrayList fieldNames = new ArrayList(); + fieldNames.add("firstInteger"); + fieldNames.add("secondString"); + fieldNames.add("thirdBoolean"); + ArrayList fieldObjectInspectors = new ArrayList(); + fieldObjectInspectors + .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector); + fieldObjectInspectors + .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); + fieldObjectInspectors + .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector); + + ArrayList fieldNames2 = new ArrayList(); + fieldNames2.add("firstString"); + fieldNames2.add("secondInteger"); + fieldNames2.add("thirdBoolean"); + ArrayList fieldObjectInspectors2 = new ArrayList(); + fieldObjectInspectors2 + .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); + fieldObjectInspectors2 + .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector); + fieldObjectInspectors2 + .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector); + + Converter unionConverter0 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors), + ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2)); + + Object convertedObject0 = unionConverter0.convert(new StandardUnionObjectInspector.StandardUnion((byte)0, 1)); + List expectedObject0 = new ArrayList(); + expectedObject0.add("1"); + + assertEquals(expectedObject0, convertedObject0); + + Converter unionConverter1 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors), + ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2)); + + Object convertedObject1 = unionConverter1.convert(new StandardUnionObjectInspector.StandardUnion((byte)1, "1")); + List expectedObject1 = new ArrayList(); + expectedObject1.add(1); + + assertEquals(expectedObject1, convertedObject1); + + Converter unionConverter2 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors), + ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2)); + + Object convertedObject2 = unionConverter2.convert(new StandardUnionObjectInspector.StandardUnion((byte)2, true)); + List expectedObject2 = new ArrayList(); + expectedObject2.add(true); + + assertEquals(expectedObject2, convertedObject2); + + // Union (extra fields) + ArrayList fieldNamesExtra = new ArrayList(); + fieldNamesExtra.add("firstInteger"); + fieldNamesExtra.add("secondString"); + fieldNamesExtra.add("thirdBoolean"); + ArrayList fieldObjectInspectorsExtra = new ArrayList(); + fieldObjectInspectorsExtra + .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector); + fieldObjectInspectorsExtra + .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); + fieldObjectInspectorsExtra + .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector); + + ArrayList fieldNamesExtra2 = new ArrayList(); + fieldNamesExtra2.add("firstString"); + fieldNamesExtra2.add("secondInteger"); + ArrayList fieldObjectInspectorsExtra2 = new ArrayList(); + fieldObjectInspectorsExtra2 + .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); + fieldObjectInspectorsExtra2 + .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector); + + Converter unionConverterExtra = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectorsExtra), + ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectorsExtra2)); + + Object convertedObjectExtra = unionConverterExtra.convert(new StandardUnionObjectInspector.StandardUnion((byte)2, true)); + List expectedObjectExtra = new ArrayList(); + expectedObjectExtra.add(null); + + assertEquals(expectedObjectExtra, convertedObjectExtra); // we should get back null + } catch (Throwable e) { e.printStackTrace(); throw e; @@ -192,4 +279,4 @@ public void testGetConvertedOI() throws Throwable { VarcharTypeInfo vcParams = (VarcharTypeInfo) poi.getTypeInfo(); assertEquals("varchar length doesn't match", 5, vcParams.getLength()); } -} +} \ No newline at end of file