Index: serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java =================================================================== --- serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java (revision 991812) +++ serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java (working copy) @@ -5,14 +5,8 @@ */ package org.apache.hadoop.hive.serde; -import java.util.List; -import java.util.ArrayList; -import java.util.Map; -import java.util.HashMap; -import java.util.Set; import java.util.HashSet; -import java.util.Collections; -import org.apache.log4j.Logger; +import java.util.Set; public class Constants { @@ -74,6 +68,8 @@ public static final String STRUCT_TYPE_NAME = "struct"; + public static final String UNION_TYPE_NAME = "union"; + public static final String LIST_COLUMNS = "columns"; public static final String LIST_COLUMN_TYPES = "columns.types"; Index: serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java (revision 991812) +++ serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java (working copy) @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; @@ -326,6 +327,20 @@ } break; } + case UNION: { + UnionObjectInspector uoi = (UnionObjectInspector) oi; + if (o == null) { + sb.append("null"); + } else { + sb.append(LBRACE); + sb.append(uoi.getTag(o)); + sb.append(COLON); + buildJSONString(sb, uoi.getField(o), + uoi.getObjectInspectors().get(uoi.getTag(o))); + sb.append(RBRACE); + } + break; + } default: throw new RuntimeException("Unknown type in ObjectInspector!"); } @@ -412,6 +427,19 @@ return false; } } + case UNION: { + UnionObjectInspector uoi = (UnionObjectInspector) oi; + if (o == null) { + return true; + } else { + // there are no elements in the union + if (uoi.getObjectInspectors().size() == 0) { + return false; + } + return hasAnyNullObject(uoi.getField(o), + uoi.getObjectInspectors().get(uoi.getTag(o))); + } + } default: throw new RuntimeException("Unknown type in ObjectInspector!"); } Index: serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java (revision 991812) +++ serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java (working copy) @@ -41,6 +41,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; @@ -56,6 +57,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.FloatWritable; @@ -390,6 +392,13 @@ } return r; } + case UNION: { + UnionTypeInfo utype = (UnionTypeInfo) type; + // Read the tag + byte tag = buffer.read(invert); + return deserialize(buffer, utype.getAllUnionObjectTypeInfos().get(tag), + invert, null); + } default: { throw new RuntimeException("Unrecognized type: " + type.getCategory()); } @@ -570,6 +579,14 @@ } return; } + case UNION: { + UnionObjectInspector uoi = (UnionObjectInspector) oi; + byte tag = uoi.getTag(o); + buffer.write(tag, invert); + serialize(buffer, uoi.getField(o), uoi.getObjectInspectors().get(tag), + invert); + return; + } default: { throw new RuntimeException("Unrecognized type: " + oi.getCategory()); } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java (revision 991812) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java (working copy) @@ -24,6 +24,7 @@ import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyObjectInspectorFactory; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyUnionObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBooleanObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyByteObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyDoubleObjectInspector; @@ -42,6 +43,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; import org.apache.hadoop.io.Text; /** @@ -92,6 +94,8 @@ return new LazyArray((LazyListObjectInspector) oi); case STRUCT: return new LazyStruct((LazySimpleStructObjectInspector) oi); + case UNION: + return new LazyUnion((LazyUnionObjectInspector) oi); } throw new RuntimeException("Hive LazySerDe Internal error."); @@ -152,6 +156,16 @@ return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector( fieldNames, fieldObjectInspectors, separator[separatorIndex], nullSequence, false, escaped, escapeChar); + case UNION: + UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; + List lazyOIs = new ArrayList(); + for (TypeInfo uti : unionTypeInfo.getAllUnionObjectTypeInfos()) { + lazyOIs.add(createLazyObjectInspector(uti, separator, + separatorIndex + 1, nullSequence, escaped, + escapeChar)); + } + return LazyObjectInspectorFactory.getLazyUnionObjectInspector(lazyOIs, + separator[separatorIndex], nullSequence, escaped, escapeChar); } throw new RuntimeException("Hive LazySerDe Internal error."); Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java (revision 991812) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java (working copy) @@ -39,6 +39,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; @@ -541,6 +542,20 @@ } } return; + case UNION: + separator = (char) separators[level]; + UnionObjectInspector uoi = (UnionObjectInspector) objInspector; + List ois = uoi.getObjectInspectors(); + if (ois == null) { + out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); + } else { + out.write(uoi.getTag(obj)); + out.write(separator); + serialize(out, uoi.getField(obj), ois.get(uoi.getTag(obj)), + separators, level + 1, nullSequence, escaped, escapeChar, + needsEscape); + } + return; default: break; } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUnion.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUnion.java (revision 0) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUnion.java (revision 0) @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.lazy; + +import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyUnionObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.io.Text; + +/** + * LazyObject for storing a union. The field of a union can be primitive or + * non-primitive. + * + */ +public class LazyUnion extends + LazyNonPrimitive { + + /** + * Whether the data is already parsed or not. + */ + private boolean parsed; + + /** + * The start position of union field. Only valid when the data is parsed. + */ + private int startPosition; + + /** + * The object of the union. + */ + private LazyObject field; + + /** + * Tag of the Union + */ + private byte tag; + + /** + * Whether init() has been called on the field or not. + */ + private boolean fieldInited = false; + + /** + * Construct a LazyUnion object with the ObjectInspector. + */ + public LazyUnion(LazyUnionObjectInspector oi) { + super(oi); + } + + /** + * Set the row data for this LazyUnion. + * + * @see LazyObject#init(ByteArrayRef, int, int) + */ + @Override + public void init(ByteArrayRef bytes, int start, int length) { + super.init(bytes, start, length); + parsed = false; + } + + /** + * Parse the byte[] and fill each field. + */ + @SuppressWarnings("unchecked") + private void parse() { + + byte separator = oi.getSeparator(); + boolean isEscaped = oi.isEscaped(); + byte escapeChar = oi.getEscapeChar(); + boolean tagInited = false; + + int unionByteEnd = start + length; + int fieldByteEnd = start; + byte[] bytes = this.bytes.getData(); + // Go through all bytes in the byte[] + while (fieldByteEnd < unionByteEnd) { + if (bytes[fieldByteEnd] != separator) { + if (isEscaped && bytes[fieldByteEnd] == escapeChar + && fieldByteEnd + 1 < unionByteEnd) { + // ignore the char after escape_char + fieldByteEnd += 1; + } else { + if (!tagInited) { + tag = bytes[fieldByteEnd]; + tagInited = true; + } + } + } else { // (bytes[fieldByteEnd] == separator) + // Reached the end of the tag + startPosition = fieldByteEnd + 1; + } + fieldByteEnd++; + } + field = LazyFactory.createLazyObject(oi.getObjectInspectors().get(tag)); + fieldInited = false; + parsed = true; + } + + /** + * Get the field out of the row without checking parsed. + * + * @return The value of the field + */ + private Object uncheckedGetField() { + Text nullSequence = oi.getNullSequence(); + int fieldLength = length - startPosition; + if (fieldLength != 0 && fieldLength == nullSequence.getLength() && + LazyUtils.compare(bytes.getData(), startPosition, fieldLength, + nullSequence.getBytes(), 0, nullSequence.getLength()) == 0) { + return null; + } + + if (!fieldInited) { + fieldInited = true; + field.init(bytes, startPosition, fieldLength); + } + return field.getObject(); + } + + /** + * Get the field out of the union. + * + * @return The field as a LazyObject + */ + public Object getField() { + if (!parsed) { + parse(); + } + return uncheckedGetField(); + } + + /** + * Get the tag of the union + * + * @return The tag byte + */ + public byte getTag() { + if (!parsed) { + parse(); + } + return tag; + } +} Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java (revision 991812) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java (working copy) @@ -112,6 +112,29 @@ return result; } + static HashMap, LazyUnionObjectInspector> + cachedLazyUnionObjectInspector = + new HashMap, LazyUnionObjectInspector>(); + + public static LazyUnionObjectInspector getLazyUnionObjectInspector( + List ois, byte separator, Text nullSequence, + boolean escaped, byte escapeChar) { + ArrayList signature = new ArrayList(); + signature.add(ois); + signature.add(Byte.valueOf(separator)); + signature.add(nullSequence.toString()); + signature.add(Boolean.valueOf(escaped)); + signature.add(Byte.valueOf(escapeChar)); + LazyUnionObjectInspector result = cachedLazyUnionObjectInspector + .get(signature); + if (result == null) { + result = new LazyUnionObjectInspector(ois, separator, + nullSequence, escaped, escapeChar); + cachedLazyUnionObjectInspector.put(signature, result); + } + return result; + } + private LazyObjectInspectorFactory() { // prevent instantiation } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyUnionObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyUnionObjectInspector.java (revision 0) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyUnionObjectInspector.java (revision 0) @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.lazy.objectinspector; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.serde2.lazy.LazyUnion; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; +import org.apache.hadoop.io.Text; + +/** + * LazyUnionObjectInspector works on union data that is stored in LazyUnion. + * + * Always use the {@link LazyObjectInspectorFactory} to create new + * ObjectInspector objects, instead of directly creating an instance of this + * class. + */ +public class LazyUnionObjectInspector implements UnionObjectInspector { + + public static final Log LOG = LogFactory + .getLog(LazyUnionObjectInspector.class.getName()); + + protected List ois; + + @Override + public String getTypeName() { + return ObjectInspectorUtils.getStandardUnionTypeName(this); + } + + byte separator; + Text nullSequence; + boolean escaped; + byte escapeChar; + + protected LazyUnionObjectInspector( + List ois, byte separator, + Text nullSequence, boolean escaped, + byte escapeChar) { + init(ois, separator, + nullSequence, escaped, escapeChar); + } + + protected void init( + List ois, byte separator, + Text nullSequence, boolean escaped, + byte escapeChar) { + this.separator = separator; + this.nullSequence = nullSequence; + this.escaped = escaped; + this.escapeChar = escapeChar; + this.ois = new ArrayList(); + this.ois.addAll(ois); + } + + protected LazyUnionObjectInspector(List ois, + byte separator, Text nullSequence) { + init(ois, separator, nullSequence); + } + + protected void init(List ois, byte separator, + Text nullSequence) { + this.separator = separator; + this.nullSequence = nullSequence; + this.ois = new ArrayList(); + this.ois.addAll(ois); + } + + @Override + public final Category getCategory() { + return Category.UNION; + } + + public byte getSeparator() { + return separator; + } + + public Text getNullSequence() { + return nullSequence; + } + + public boolean isEscaped() { + return escaped; + } + + public byte getEscapeChar() { + return escapeChar; + } + + @Override + public Object getField(Object data) { + if (data == null) { + return null; + } + return ((LazyUnion) data).getField(); + } + + @Override + public List getObjectInspectors() { + return ois; + } + + @Override + public byte getTag(Object data) { + if (data == null) { + return -1; + } + return ((LazyUnion) data).getTag(); + } +} Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java (revision 991812) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java (working copy) @@ -42,7 +42,7 @@ * */ public static enum Category { - PRIMITIVE, LIST, MAP, STRUCT + PRIMITIVE, LIST, MAP, STRUCT, UNION }; /** Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java (revision 991812) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java (working copy) @@ -198,6 +198,21 @@ return result; } + static HashMap, StandardUnionObjectInspector> + cachedStandardUnionObjectInspector = + new HashMap, StandardUnionObjectInspector>(); + + public static StandardUnionObjectInspector getStandardUnionObjectInspector( + List unioinObjectInspectors) { + StandardUnionObjectInspector result = cachedStandardUnionObjectInspector + .get(unioinObjectInspectors); + if (result == null) { + result = new StandardUnionObjectInspector(unioinObjectInspectors); + cachedStandardUnionObjectInspector.put(unioinObjectInspectors, result); + } + return result; + } + static HashMap>, StandardStructObjectInspector> cachedStandardStructObjectInspector = new HashMap>, StandardStructObjectInspector>(); public static StandardStructObjectInspector getStandardStructObjectInspector( Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java (revision 991812) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java (working copy) @@ -29,6 +29,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; @@ -137,6 +138,15 @@ fieldNames, fieldObjectInspectors); break; } + case UNION: { + UnionObjectInspector uoi = (UnionObjectInspector) oi; + List ois = new ArrayList(); + for (ObjectInspector eoi : uoi.getObjectInspectors()) { + ois.add(getStandardObjectInspector(eoi, objectInspectorOption)); + } + result = ObjectInspectorFactory.getStandardUnionObjectInspector(ois); + break; + } default: { throw new RuntimeException("Unknown ObjectInspector category!"); } @@ -243,6 +253,16 @@ result = struct; break; } + case UNION: { + UnionObjectInspector uoi = (UnionObjectInspector)oi; + List objectInspectors = uoi.getObjectInspectors(); + Object object = copyToStandardObject( + uoi.getField(o), + objectInspectors.get(uoi.getTag(o)), + objectInspectorOption); + result = object; + break; + } default: { throw new RuntimeException("Unknown ObjectInspector category!"); } @@ -266,6 +286,20 @@ return sb.toString(); } + public static String getStandardUnionTypeName(UnionObjectInspector uoi) { + StringBuilder sb = new StringBuilder(); + sb.append(Constants.UNION_TYPE_NAME + "<"); + List ois = uoi.getObjectInspectors(); + for(int i = 0; i < ois.size(); i++) { + if (i > 0) { + sb.append(","); + } + sb.append(ois.get(i).getTypeName()); + } + sb.append(">"); + return sb.toString(); + } + public static StructField getStandardStructFieldRef(String fieldName, List fields) { fieldName = fieldName.toLowerCase(); @@ -344,6 +378,20 @@ } return result.toString(); } + case UNION: { + StringBuffer result = new StringBuffer(); + result.append(oi.getClass().getSimpleName() + "<"); + UnionObjectInspector uoi = (UnionObjectInspector)oi; + List ois = uoi.getObjectInspectors(); + for (int i = 0; i < ois.size(); i++) { + if (i > 0) { + result.append(","); + } + result.append(getObjectInspectorName(ois.get(i))); + } + result.append(">"); + return result.toString(); + } default: { throw new RuntimeException("Unknown ObjectInspector category!"); } @@ -400,6 +448,7 @@ case STRUCT: case LIST: case MAP: + case UNION: default: throw new RuntimeException( "Hash code on complex types not supported yet."); @@ -447,6 +496,14 @@ return true; case MAP: return false; + case UNION: + UnionObjectInspector uoi = (UnionObjectInspector) oi; + for (ObjectInspector eoi : uoi.getObjectInspectors()) { + if (!compareSupported(eoi)) { + return false; + } + } + return true; default: return false; } @@ -565,6 +622,13 @@ case MAP: { throw new RuntimeException("Compare on map type not supported!"); } + case UNION: { + UnionObjectInspector uoi1 = (UnionObjectInspector) oi1; + UnionObjectInspector uoi2 = (UnionObjectInspector) oi2; + return compare(uoi1.getField(o1), + uoi1.getObjectInspectors().get(uoi1.getTag(o1)), + uoi2.getField(o2), uoi2.getObjectInspectors().get(uoi2.getTag(o1))); + } default: throw new RuntimeException("Compare on unknown type: " + oi1.getCategory()); @@ -714,6 +778,29 @@ return true; } + if (c1.equals(Category.UNION)) { + UnionObjectInspector uoi1 = (UnionObjectInspector) o1; + UnionObjectInspector uoi2 = (UnionObjectInspector) o2; + List ois1 = uoi1.getObjectInspectors(); + List ois2 = uoi2.getObjectInspectors(); + + if (ois1 == null && ois2 == null) { + return true; + } else if (ois1 == null || ois2 == null) { + return false; + } else if (ois1.size() != ois2.size()) { + return false; + } + Iterator it1 = ois1.iterator(); + Iterator it2 = ois2.iterator(); + while (it1.hasNext()) { + if (!compareTypes(it1.next(), it2.next())) { + return false; + } + } + return true; + } + // Unknown category throw new RuntimeException("Unknown category encountered: " + c1); } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java (revision 0) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java (revision 0) @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.objectinspector; + +import java.util.List; + +/** + * StandardUnionObjectInspector works on union data that is stored as + * UnionObject. + * It holds the list of the object inspectors corresponding to each type of the + * object the Union can hold. The UniobObject has tag followed by the object + * it is holding. + * + * Always use the {@link ObjectInspectorFactory} to create new ObjectInspector + * objects, instead of directly creating an instance of this class. + */ +public class StandardUnionObjectInspector implements UnionObjectInspector { + List ois; + + public StandardUnionObjectInspector(List ois) { + this.ois = ois; + } + + public List getObjectInspectors() { + return ois; + } + + public static class MyUnion implements UnionObject { + protected byte tag; + protected Object object; + + public MyUnion(byte tag, Object object) { + this.tag = tag; + this.object = object; + } + + @Override + public Object getObject() { + return object; + } + + @Override + public byte getTag() { + return tag; + } + } + + /** + * Return the tag of the object. + */ + public byte getTag(Object o) { + if (o == null) { + return -1; + } + return ((UnionObject) o).getTag(); + } + + /** + * Return the field based on the tag value associated with the Object. + */ + public Object getField(Object o) { + if (o == null) { + return null; + } + return ((UnionObject) o).getObject(); + } + + public Category getCategory() { + return Category.UNION; + } + + public String getTypeName() { + return ObjectInspectorUtils.getStandardUnionTypeName(this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getName()); + sb.append(getTypeName()); + return sb.toString(); + } + +} Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionObject.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionObject.java (revision 0) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionObject.java (revision 0) @@ -0,0 +1,24 @@ +package org.apache.hadoop.hive.serde2.objectinspector; + +/** + * The UnionObject. + * + * It has tag followed by the object it is holding. + * + */ +public interface UnionObject { + /** + * Get the tag of the union. + * + * @return the tag byte + */ + byte getTag(); + + /** + * Get the Object. + * + * @return The Object union is holding + */ + Object getObject(); + +} Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionObjectInspector.java (revision 0) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionObjectInspector.java (revision 0) @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.objectinspector; + +import java.util.List; + +/** + * UnionObjectInspector works on union data that is stored as UnionObject. + * + * It holds the list of the object inspectors corresponding to each type of the + * object the Union can hold. + * + * UnionObjectInspector. + * + */ +public interface UnionObjectInspector extends ObjectInspector { + + /** + * Returns the array of ObjectInspectors that are for each of the tags. + */ + List getObjectInspectors(); + + /** + * Return the tag of the object. + */ + byte getTag(Object o); + + /** + * Return the field based on the tag associated with the Object. + */ + Object getField(Object o); + +} Index: serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java (revision 991812) +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java (working copy) @@ -25,11 +25,11 @@ /** * Stores information about a type. Always use the TypeInfoFactory to create new * TypeInfo objects. - * - * We support 4 categories of types: 1. Primitive objects (String, Number, etc) + * + * We support 5 categories of types: 1. Primitive objects (String, Number, etc) * 2. List objects (a list of objects of a single type) 3. Map objects (a map * from objects of one type to objects of another type) 4. Struct objects (a - * list of fields with names and their own types) + * list of fields with names and their own types) 5. Union objects */ public abstract class TypeInfo implements Serializable { @@ -39,8 +39,8 @@ } /** - * The Category of this TypeInfo. Possible values are Primitive, List, Map and - * Struct, which corresponds to the 4 sub-classes of TypeInfo. + * The Category of this TypeInfo. Possible values are Primitive, List, Map, + * Struct and Union, which corresponds to the 5 sub-classes of TypeInfo. */ public abstract Category getCategory(); Index: serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java (revision 991812) +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java (working copy) @@ -97,6 +97,18 @@ return result; } + static HashMap, TypeInfo> cachedUnionTypeInfo = + new HashMap, TypeInfo>(); + + public static TypeInfo getUnionTypeInfo(List typeInfos) { + TypeInfo result = cachedUnionTypeInfo.get(typeInfos); + if (result == null) { + result = new UnionTypeInfo(typeInfos); + cachedUnionTypeInfo.put(typeInfos, result); + } + return result; + } + static HashMap cachedListTypeInfo = new HashMap(); public static TypeInfo getListTypeInfo(TypeInfo elementTypeInfo) { Index: serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java (revision 991812) +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java (working copy) @@ -19,6 +19,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; @@ -276,6 +277,7 @@ if (!Constants.LIST_TYPE_NAME.equals(t.text) && !Constants.MAP_TYPE_NAME.equals(t.text) && !Constants.STRUCT_TYPE_NAME.equals(t.text) + && !Constants.UNION_TYPE_NAME.equals(t.text) && null == PrimitiveObjectInspectorUtils .getTypeEntryFromTypeName(t.text) && !t.text.equals(alternative)) { @@ -354,6 +356,26 @@ return TypeInfoFactory.getStructTypeInfo(fieldNames, fieldTypeInfos); } + // Is this a union type? + if (Constants.UNION_TYPE_NAME.equals(t.text)) { + ArrayList objectTypeInfos = new ArrayList(); + boolean first = true; + do { + if (first) { + expect("<"); + first = false; + } else { + Token separator = expect(">", ","); + if (separator.text.equals(">")) { + // end of union + break; + } + } + objectTypeInfos.add(parseType()); + } while (true); + + return TypeInfoFactory.getUnionTypeInfo(objectTypeInfos); + } throw new RuntimeException("Internal error parsing position " + t.position + " of '" + typeInfoString + "'"); @@ -413,6 +435,22 @@ fieldNames, fieldObjectInspectors); break; } + case UNION: { + UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; + List objectTypeInfos = unionTypeInfo + .getAllUnionObjectTypeInfos(); + List fieldObjectInspectors = + new ArrayList(objectTypeInfos.size()); + for (int i = 0; i < objectTypeInfos.size(); i++) { + fieldObjectInspectors + .add(getStandardWritableObjectInspectorFromTypeInfo(objectTypeInfos + .get(i))); + } + result = ObjectInspectorFactory.getStandardUnionObjectInspector( + fieldObjectInspectors); + break; + } + default: { result = null; } @@ -476,7 +514,22 @@ fieldNames, fieldObjectInspectors); break; } - default: { + case UNION: { + UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; + List objectTypeInfos = unionTypeInfo + .getAllUnionObjectTypeInfos(); + List fieldObjectInspectors = + new ArrayList(objectTypeInfos.size()); + for (int i = 0; i < objectTypeInfos.size(); i++) { + fieldObjectInspectors + .add(getStandardJavaObjectInspectorFromTypeInfo(objectTypeInfos + .get(i))); + } + result = ObjectInspectorFactory.getStandardUnionObjectInspector( + fieldObjectInspectors); + break; + } + default: { result = null; } } @@ -532,6 +585,15 @@ result = TypeInfoFactory.getStructTypeInfo(fieldNames, fieldTypeInfos); break; } + case UNION: { + UnionObjectInspector uoi = (UnionObjectInspector) oi; + List objectTypeInfos = new ArrayList(); + for (ObjectInspector eoi : uoi.getObjectInspectors()) { + objectTypeInfos.add(getTypeInfoFromObjectInspector(eoi)); + } + result = TypeInfoFactory.getUnionTypeInfo(objectTypeInfos); + break; + } default: { throw new RuntimeException("Unknown ObjectInspector category!"); } Index: serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnionTypeInfo.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnionTypeInfo.java (revision 0) +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnionTypeInfo.java (revision 0) @@ -0,0 +1,87 @@ +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; + +/** + * UnionTypeInfo represents the TypeInfo of an union. A union holds only one + * field of the specified fields at any point of time. The fields, a Union can + * hold, can have the same or different TypeInfo. + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. + */ +public class UnionTypeInfo extends TypeInfo implements Serializable { + + private static final long serialVersionUID = 1L; + + private ArrayList allUnionObjectTypeInfos; + + /** + * For java serialization use only. + */ + public UnionTypeInfo() { + } + + @Override + public String getTypeName() { + StringBuilder sb = new StringBuilder(); + sb.append(Constants.UNION_TYPE_NAME + "<"); + for (int i = 0; i < allUnionObjectTypeInfos.size(); i++) { + if (i > 0) { + sb.append(","); + } + sb.append(allUnionObjectTypeInfos.get(i).getTypeName()); + } + sb.append(">"); + return sb.toString(); + } + + /** + * For java serialization use only. + */ + public void setAllUnionObjectTypeInfos( + ArrayList allUnionObjectTypeInfos) { + this.allUnionObjectTypeInfos = allUnionObjectTypeInfos; + } + + /** + * For TypeInfoFactory use only. + */ + UnionTypeInfo(List typeInfos) { + allUnionObjectTypeInfos = new ArrayList(); + allUnionObjectTypeInfos.addAll(typeInfos); + } + + @Override + public Category getCategory() { + return Category.UNION; + } + + public ArrayList getAllUnionObjectTypeInfos() { + return allUnionObjectTypeInfos; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof UnionTypeInfo)) { + return false; + } + UnionTypeInfo o = (UnionTypeInfo) other; + + // Compare the field types + return o.getAllUnionObjectTypeInfos().equals(getAllUnionObjectTypeInfos()); + } + + @Override + public int hashCode() { + return allUnionObjectTypeInfos.hashCode(); + } +} Index: serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java (revision 991812) +++ serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java (working copy) @@ -32,7 +32,7 @@ import org.apache.hadoop.io.Text; /** - * TestLazyArrayMapStruct. + * Tests LazyArray, LazyMap, LazyStruct and LazyUnion * */ public class TestLazyArrayMapStruct extends TestCase { @@ -244,4 +244,80 @@ } } + private byte[] getUnionBytes(byte tag, Text data) { + byte[] bytes = new byte[data.getBytes().length + 1]; + bytes[0] = tag; + int i = 1; + for (byte b: data.getBytes()) { + bytes[i++] = b; + } + return bytes; + } + + /** + * Test the LazyUnion class. + */ + public void testLazyUnion() throws Throwable { + try { + { + TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString( + "union,map,string>"); + Text nullSequence = new Text("\\N"); + + ObjectInspector oi = LazyFactory.createLazyObjectInspector(typeInfo, + new byte[] {'^', ':', '='}, 0, nullSequence, false, (byte) 0); + LazyUnion o = (LazyUnion) LazyFactory.createLazyObject(oi); + + Text data; + + data = new Text("^123"); + TestLazyPrimitive.initLazyObject(o, getUnionBytes((byte)0, data), 0, + data.getLength() + 1); + assertEquals("{0:123}", SerDeUtils.getJSONString(o, oi)); + + data = new Text("^a:b:c"); + TestLazyPrimitive.initLazyObject(o, getUnionBytes((byte)1, data), 0, + data.getLength() + 1); + assertEquals("{1:[\"a\",\"b\",\"c\"]}", SerDeUtils.getJSONString(o, oi)); + + data = new Text("^d=e:f=g"); + TestLazyPrimitive.initLazyObject(o, getUnionBytes((byte)2, data), 0, + data.getLength() + 1); + assertEquals( + "{2:{\"d\":\"e\",\"f\":\"g\"}}", SerDeUtils.getJSONString(o, oi)); + + data = new Text("^hi"); + TestLazyPrimitive.initLazyObject(o, getUnionBytes((byte)3, data), 0, + data.getLength() + 1) ; + assertEquals("{3:\"hi\"}", SerDeUtils.getJSONString(o, oi)); + + + data = new Text("^\\N"); + TestLazyPrimitive.initLazyObject(o, getUnionBytes((byte)0, data), 0, + data.getLength() + 1); + assertEquals("{0:null}", SerDeUtils.getJSONString(o, oi)); + + data = new Text("^ :a::"); + TestLazyPrimitive.initLazyObject(o, getUnionBytes((byte)1, data), 0, + data.getLength() + 1); + assertEquals( + "{1:[\" \",\"a\",\"\",\"\"]}", SerDeUtils.getJSONString(o, oi)); + + data = new Text("^d=\\N:f=g:h"); + TestLazyPrimitive.initLazyObject(o, getUnionBytes((byte)2, data), 0, + data.getLength() + 1 ); + assertEquals( + "{2:{\"d\":null,\"f\":\"g\",\"h\":null}}", + SerDeUtils.getJSONString(o, oi)); + + data = new Text("^= "); + TestLazyPrimitive.initLazyObject(o, getUnionBytes((byte)2, data), 0, + data.getLength() + 1); + assertEquals("{2:{\"\":\" \"}}", SerDeUtils.getJSONString(o, oi)); + } + } catch (Throwable e) { + e.printStackTrace(); + throw e; + } + } } Index: serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java (revision 991812) +++ serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java (working copy) @@ -23,12 +23,17 @@ import junit.framework.TestCase; +import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.MyUnion; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.FloatWritable; import org.apache.hadoop.io.IntWritable; @@ -334,4 +339,182 @@ } + @SuppressWarnings("unchecked") + public void testStandardUnionObjectInspector() throws Throwable { + try { + ArrayList objectInspectors = new ArrayList(); + // add primitive types + objectInspectors + .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector); + objectInspectors + .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); + objectInspectors + .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector); + + // add a list + objectInspectors + .add(ObjectInspectorFactory + .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector)); + + // add a map + objectInspectors + .add(ObjectInspectorFactory + .getStandardMapObjectInspector( + PrimitiveObjectInspectorFactory.javaIntObjectInspector, + PrimitiveObjectInspectorFactory.javaStringObjectInspector)); + + // add a struct + ArrayList fieldNames = new ArrayList(); + fieldNames.add("myDouble"); + fieldNames.add("myLong"); + ArrayList fieldObjectInspectors = new ArrayList(); + fieldObjectInspectors + .add(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector); + fieldObjectInspectors + .add(PrimitiveObjectInspectorFactory.javaLongObjectInspector); + objectInspectors + .add(ObjectInspectorFactory + .getStandardStructObjectInspector(fieldNames, fieldObjectInspectors)); + + StandardUnionObjectInspector uoi1 = ObjectInspectorFactory + .getStandardUnionObjectInspector(objectInspectors); + StandardUnionObjectInspector uoi2 = ObjectInspectorFactory + .getStandardUnionObjectInspector( + (ArrayList) objectInspectors.clone()); + assertEquals(uoi1, uoi2); + assertEquals(ObjectInspectorUtils.getObjectInspectorName(uoi1), + ObjectInspectorUtils.getObjectInspectorName(uoi2)); + assertTrue(ObjectInspectorUtils.compareTypes(uoi1, uoi2)); + // compareSupported returns false because Union can contain + // an object of Map + assertFalse(ObjectInspectorUtils.compareSupported(uoi1)); + + // construct unionObjectInspector without Map field. + ArrayList ois = + (ArrayList)objectInspectors.clone(); + ois.set(4, PrimitiveObjectInspectorFactory.javaIntObjectInspector); + assertTrue(ObjectInspectorUtils.compareSupported(ObjectInspectorFactory + .getStandardUnionObjectInspector(ois))); + + // metadata + assertEquals(Category.UNION, uoi1.getCategory()); + List uois = uoi1.getObjectInspectors(); + assertEquals(6, uois.size()); + for (int i = 0; i < 6; i++) { + assertEquals(objectInspectors.get(i), uois.get(i)); + } + StringBuilder unionTypeName = new StringBuilder(); + unionTypeName.append("union<"); + for (int i = 0; i < uois.size(); i++) { + if (i > 0) { + unionTypeName.append(","); + } + unionTypeName.append(uois.get(i).getTypeName()); + } + unionTypeName.append(">"); + assertEquals(unionTypeName.toString(), uoi1.getTypeName()); + // TypeInfo + TypeInfo typeInfo1 = TypeInfoUtils.getTypeInfoFromObjectInspector(uoi1); + assertEquals(Category.UNION, typeInfo1.getCategory()); + assertEquals(UnionTypeInfo.class.getName(), typeInfo1.getClass().getName()); + assertEquals(typeInfo1.getTypeName(), uoi1.getTypeName()); + assertEquals(typeInfo1, + TypeInfoUtils.getTypeInfoFromTypeString(uoi1.getTypeName())); + assertEquals(typeInfo1, + TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo1)); + TypeInfo typeInfo2 = TypeInfoUtils.getTypeInfoFromObjectInspector(uoi2); + assertEquals(typeInfo1, typeInfo2); + assertEquals(typeInfo2, TypeInfoUtils. + getStandardJavaObjectInspectorFromTypeInfo(typeInfo2)); + assertEquals(TypeInfoUtils. + getStandardWritableObjectInspectorFromTypeInfo(typeInfo1), + TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo( + typeInfo2)); + + // null + assertNull(uoi1.getField(null)); + assertEquals(-1, uoi1.getTag(null)); + + // Union + UnionObject union = new MyUnion((byte)0, 1); + assertEquals(0, uoi1.getTag(union)); + assertEquals(1, uoi1.getField(union)); + assertEquals("{0:1}", SerDeUtils.getJSONString(union, uoi1)); + assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, + new MyUnion((byte)0, 1), uoi2)); + assertTrue( + ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(1)); + + union = new MyUnion((byte)1, "two"); + assertEquals(1, uoi1.getTag(union)); + assertEquals("two", uoi1.getField(union)); + assertEquals("{1:\"two\"}", SerDeUtils.getJSONString(union, uoi1)); + assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, + new MyUnion((byte)1, "two"), uoi2)); + assertTrue( + ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals("two")); + + union = new MyUnion((byte)2, true); + assertEquals(2, uoi1.getTag(union)); + assertEquals(true, uoi1.getField(union)); + assertEquals("{2:true}", SerDeUtils.getJSONString(union, uoi1)); + assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, + new MyUnion((byte)2, true), uoi2)); + assertTrue( + ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(true)); + + ArrayList iList = new ArrayList(); + iList.add(4); + iList.add(5); + union = new MyUnion((byte)3, iList); + assertEquals(3, uoi1.getTag(union)); + assertEquals(iList, uoi1.getField(union)); + assertEquals("{3:[4,5]}", SerDeUtils.getJSONString(union, uoi1)); + assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, + new MyUnion((byte)3, iList.clone()), uoi2)); + assertTrue( + ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(iList)); + + HashMap map = new HashMap(); + map.put(6, "six"); + map.put(7, "seven"); + map.put(8, "eight"); + union = new MyUnion((byte)4, map); + assertEquals(4, uoi1.getTag(union)); + assertEquals(map, uoi1.getField(union)); + assertEquals("{4:{6:\"six\",7:\"seven\",8:\"eight\"}}", + SerDeUtils.getJSONString(union, uoi1)); + Throwable th = null; + try { + ObjectInspectorUtils.compare(union, uoi1, + new MyUnion((byte)4, map.clone()), uoi2); + } catch (Throwable t) { + th = t; + } + assertNotNull(th); + assertEquals("Compare on map type not supported!",th.getMessage()); + assertTrue( + ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(map)); + + + ArrayList struct = new ArrayList(2); + struct.add(9.0); + struct.add(10L); + union = new MyUnion((byte)5, struct); + assertEquals(5, uoi1.getTag(union)); + assertEquals(struct, uoi1.getField(union)); + assertEquals("{5:{\"mydouble\":9.0,\"mylong\":10}}", + SerDeUtils.getJSONString(union, uoi1)); + assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, + new MyUnion((byte)5, struct.clone()), uoi2)); + assertTrue( + ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(struct)); + + } catch (Throwable e) { + e.printStackTrace(); + throw e; + } + + } + }