Index: serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java (revision 1352373) +++ serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java (working copy) @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.serde2.lazybinary; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -35,14 +36,22 @@ import org.apache.hadoop.hive.serde2.binarysortable.MyTestInnerStruct; import org.apache.hadoop.hive.serde2.binarysortable.TestBinarySortableSerDe; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; +import org.apache.hadoop.hive.serde2.lazy.LazyBinary; +import org.apache.hadoop.hive.serde2.lazy.LazyFactory; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.AbstractPrimitiveLazyObjectInspector; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryMapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaBinaryObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBinaryObjectInspector; import org.apache.hadoop.io.BytesWritable; /** @@ -53,7 +62,7 @@ /** * Generate a random struct array. - * + * * @param r * random number generator * @return an struct array @@ -71,7 +80,7 @@ /** * Initialize the LazyBinarySerDe. - * + * * @param fieldNames * table field names * @param fieldTypes @@ -91,7 +100,7 @@ /** * Test the LazyBinarySerDe. - * + * * @param rows * array of structs to be serialized * @param rowOI @@ -134,7 +143,7 @@ * Compare two structs that have different number of fields. We just compare * the first few common fields, ignoring the fields existing in one struct but * not the other. - * + * * @see ObjectInspectorUtils#compare(Object, ObjectInspector, Object, * ObjectInspector) */ @@ -464,7 +473,7 @@ /** * The test entrance function. - * + * * @throws Throwable */ public void testLazyBinarySerDe() throws Throwable { @@ -522,4 +531,78 @@ throw e; } } + + private final byte[] inpBArray = {'1','\u0001','3','4'}; + private BytesWritable getInputBytesWritable() { + //create input BytesWritable. This would have capacity greater than length) + BytesWritable bW = new BytesWritable(); + bW.set(inpBArray, 0, inpBArray.length); + return bW; + } + + /** + * Test to see if ByteArrayRef with correct contents is generated by + * JavaBinaryObjectInspector from input BytesWritable + * @throws Throwable + */ + public void testJavaBinaryObjectInspector() throws Throwable { + BytesWritable bW = getInputBytesWritable(); + + //create JavaBinaryObjectInspector + JavaBinaryObjectInspector binInspector = + PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector; + + //convert BytesWritable to ByteArrayRef + ByteArrayRef outBARef = binInspector.set(null, bW); + + assertTrue("compare input and output BAs", + Arrays.equals(inpBArray, outBARef.getData())); + } + + + /** + * Test to see if ByteArrayRef with correct contents is generated by + * WritableBinaryObjectInspector from input BytesWritable + * @throws Throwable + */ + public void testWritableBinaryObjectInspector() throws Throwable { + BytesWritable bW = getInputBytesWritable(); + + //test WritableBinaryObjectInspector + WritableBinaryObjectInspector writableBinInsp = + PrimitiveObjectInspectorFactory.writableBinaryObjectInspector; + + //convert BytesWritable to ByteArrayRef + ByteArrayRef outBARef = writableBinInsp.getPrimitiveJavaObject(bW); + + assertTrue("compare input and output BAs", + Arrays.equals(inpBArray, outBARef.getData())); + } + + /** + * Test to see if ByteArrayRef with correct contents is generated by + * LazyBinaryObjectInspector from input BytesWritable + * @throws Throwable + */ + public void testLazyBinaryObjectInspector() throws Throwable { + + //create input ByteArrayRef + ByteArrayRef inpBARef = new ByteArrayRef(); + inpBARef.setData(inpBArray); + + AbstractPrimitiveLazyObjectInspector binInspector = LazyPrimitiveObjectInspectorFactory + .getLazyObjectInspector(PrimitiveCategory.BINARY, false, (byte)0); + + //create LazyBinary initialed with inputBA + LazyBinary lazyBin = (LazyBinary) LazyFactory.createLazyObject(binInspector); + lazyBin.init(inpBARef, 0, inpBArray.length); + + //use inspector to get a ByteArrayRef out of LazyBinary + ByteArrayRef outBARef = (ByteArrayRef) binInspector.getPrimitiveJavaObject(lazyBin); + + assertTrue("compare input and output BAs", + Arrays.equals(inpBArray, outBARef.getData())); + + } + } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (revision 1352373) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (working copy) @@ -349,6 +349,29 @@ } } + /** + * Creates a ByteArrayRef with data from source BytesWritable + * If the length of underlying byte[] of sourceBw is same as + * sourceBw.getLength(), it does not do a copy of the byte[]. + * Make sure its OK to re-use the underlying byte[] ! + * @param sourceBw - source BytesWritable + */ + public static ByteArrayRef createByteArrayRef(BytesWritable sourceBw){ + byte[] newData; + if(sourceBw.getBytes().length == sourceBw.getLength()){ + newData = sourceBw.getBytes(); + }else{ + //TODO should use BytesWritable.copyData() here once Hive + // removes support for the Hadoop 0.20 series. + newData = new byte[sourceBw.getLength()]; + System.arraycopy(sourceBw.getBytes(), 0, newData, 0, sourceBw.getLength()); + } + + ByteArrayRef bar = new ByteArrayRef(); + bar.setData(newData); + return bar; + } + private LazyUtils() { // prevent instantiation } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBinaryObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBinaryObjectInspector.java (revision 1352373) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBinaryObjectInspector.java (working copy) @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; import org.apache.hadoop.hive.serde2.lazy.LazyBinary; +import org.apache.hadoop.hive.serde2.lazy.LazyUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.io.BytesWritable; @@ -42,9 +43,7 @@ if (null == o) { return null; } - ByteArrayRef ba = new ByteArrayRef(); - ba.setData(((LazyBinary) o).getWritableObject().getBytes()); - return ba; + return LazyUtils.createByteArrayRef(((LazyBinary) o).getWritableObject()); } @Override Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBinaryObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBinaryObjectInspector.java (revision 1352373) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBinaryObjectInspector.java (working copy) @@ -1,6 +1,7 @@ package org.apache.hadoop.hive.serde2.objectinspector.primitive; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; +import org.apache.hadoop.hive.serde2.lazy.LazyUtils; import org.apache.hadoop.io.BytesWritable; /** @@ -63,9 +64,7 @@ if (null == bw){ return null; } - ByteArrayRef ba = (ByteArrayRef)o; - ba.setData(bw.getBytes()); - return ba; + return LazyUtils.createByteArrayRef(bw); } @Override @@ -80,9 +79,7 @@ if(null == bw){ return null; } - ByteArrayRef ba = new ByteArrayRef(); - ba.setData(bw.getBytes()); - return ba; + return LazyUtils.createByteArrayRef(bw); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBinaryObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBinaryObjectInspector.java (revision 1352373) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBinaryObjectInspector.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.serde2.objectinspector.primitive; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; +import org.apache.hadoop.hive.serde2.lazy.LazyUtils; import org.apache.hadoop.io.BytesWritable; public class WritableBinaryObjectInspector extends AbstractPrimitiveWritableObjectInspector @@ -44,9 +45,7 @@ if (null == o){ return null; } - ByteArrayRef ba = new ByteArrayRef(); - ba.setData(((BytesWritable)o).getBytes()); - return ba; + return LazyUtils.createByteArrayRef((BytesWritable)o); } @Override