Index: serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java (working copy) @@ -17,25 +17,20 @@ */ package org.apache.hadoop.hive.serde2.lazy; - import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Map; +import junit.framework.TestCase; + import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.io.ByteWritable; -import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; -import java.util.HashMap; -import junit.framework.TestCase; - public class TestLazyArrayMapStruct extends TestCase { /** @@ -45,51 +40,63 @@ try { // Array of Byte Text nullSequence = new Text("\\N"); - ObjectInspector oi = LazyFactory.createLazyObjectInspector( - TypeInfoUtils.getTypeInfosFromTypeString("array").get(0), - new byte[]{(byte)1}, 0, nullSequence, false, (byte)0); - LazyArray b = (LazyArray)LazyFactory.createLazyObject(oi); - byte[] data = new byte[]{'-', '1', 1, '\\', 'N', 1, '8'}; + ObjectInspector oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils + .getTypeInfosFromTypeString("array").get(0), + new byte[] { (byte) 1 }, 0, nullSequence, false, (byte) 0); + LazyArray b = (LazyArray) LazyFactory.createLazyObject(oi); + byte[] data = new byte[] { '-', '1', 1, '\\', 'N', 1, '8' }; TestLazyPrimitive.initLazyObject(b, data, 0, data.length); - + assertNull(b.getListElementObject(-1)); - assertEquals(new ByteWritable((byte)-1), ((LazyByte)b.getListElementObject(0)).getWritableObject()); - assertEquals(new ByteWritable((byte)-1), ((LazyByte)b.getList().get(0)).getWritableObject()); + assertEquals(new ByteWritable((byte) -1), ((LazyByte) b + .getListElementObject(0)).getWritableObject()); + assertEquals(new ByteWritable((byte) -1), ((LazyByte) b.getList().get(0)) + .getWritableObject()); assertNull(b.getListElementObject(1)); assertNull(b.getList().get(1)); - assertEquals(new ByteWritable((byte)8), ((LazyByte)b.getListElementObject(2)).getWritableObject()); - assertEquals(new ByteWritable((byte)8), ((LazyByte)b.getList().get(2)).getWritableObject()); + assertEquals(new ByteWritable((byte) 8), ((LazyByte) b + .getListElementObject(2)).getWritableObject()); + assertEquals(new ByteWritable((byte) 8), ((LazyByte) b.getList().get(2)) + .getWritableObject()); assertNull(b.getListElementObject(3)); assertEquals(3, b.getList().size()); - + // Array of String - oi = LazyFactory.createLazyObjectInspector( - TypeInfoUtils.getTypeInfosFromTypeString("array").get(0), - new byte[]{(byte)'\t'}, 0, nullSequence, false, (byte)0); - b = (LazyArray)LazyFactory.createLazyObject(oi); - data = new byte[]{'a', 'b', '\t', 'c', '\t', '\\', 'N', '\t', '\t', 'd'}; + oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils + .getTypeInfosFromTypeString("array").get(0), + new byte[] { (byte) '\t' }, 0, nullSequence, false, (byte) 0); + b = (LazyArray) LazyFactory.createLazyObject(oi); + data = new byte[] { 'a', 'b', '\t', 'c', '\t', '\\', 'N', '\t', '\t', 'd' }; // Note: the first and last element of the byte[] are NOT used TestLazyPrimitive.initLazyObject(b, data, 1, data.length - 2); assertNull(b.getListElementObject(-1)); - assertEquals(new Text("b"), ((LazyString)b.getListElementObject(0)).getWritableObject()); - assertEquals(new Text("b"), ((LazyString)b.getList().get(0)).getWritableObject()); - assertEquals(new Text("c"), ((LazyString)b.getListElementObject(1)).getWritableObject()); - assertEquals(new Text("c"), ((LazyString)b.getList().get(1)).getWritableObject()); - assertNull(((LazyString)b.getListElementObject(2))); - assertNull(((LazyString)b.getList().get(2))); - assertEquals(new Text(""), ((LazyString)b.getListElementObject(3)).getWritableObject()); - assertEquals(new Text(""), ((LazyString)b.getList().get(3)).getWritableObject()); - assertEquals(new Text(""), ((LazyString)b.getListElementObject(4)).getWritableObject()); - assertEquals(new Text(""), ((LazyString)b.getList().get(4)).getWritableObject()); - assertNull(((LazyString)b.getListElementObject(5))); + assertEquals(new Text("b"), ((LazyString) b.getListElementObject(0)) + .getWritableObject()); + assertEquals(new Text("b"), ((LazyString) b.getList().get(0)) + .getWritableObject()); + assertEquals(new Text("c"), ((LazyString) b.getListElementObject(1)) + .getWritableObject()); + assertEquals(new Text("c"), ((LazyString) b.getList().get(1)) + .getWritableObject()); + assertNull((b.getListElementObject(2))); + assertNull((b.getList().get(2))); + assertEquals(new Text(""), ((LazyString) b.getListElementObject(3)) + .getWritableObject()); + assertEquals(new Text(""), ((LazyString) b.getList().get(3)) + .getWritableObject()); + assertEquals(new Text(""), ((LazyString) b.getListElementObject(4)) + .getWritableObject()); + assertEquals(new Text(""), ((LazyString) b.getList().get(4)) + .getWritableObject()); + assertNull((b.getListElementObject(5))); assertEquals(5, b.getList().size()); - + } catch (Throwable e) { e.printStackTrace(); throw e; } } - + /** * Test the LazyMap class. */ @@ -98,114 +105,140 @@ { // Map of Integer to String Text nullSequence = new Text("\\N"); - ObjectInspector oi = LazyFactory.createLazyObjectInspector( - TypeInfoUtils.getTypeInfosFromTypeString("map").get(0), - new byte[]{(byte)1, (byte)2}, 0, nullSequence, false, (byte)0); - LazyMap b = (LazyMap)LazyFactory.createLazyObject(oi); - byte[] data = new byte[]{'2', 2, 'd', 'e', 'f', 1, '-', '1', 2, '\\', 'N', 1, '0', 2, '0', 1, '8', 2, 'a', 'b', 'c'}; + ObjectInspector oi = LazyFactory + .createLazyObjectInspector(TypeInfoUtils + .getTypeInfosFromTypeString("map").get(0), + new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, + (byte) 0); + LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi); + byte[] data = new byte[] { '2', 2, 'd', 'e', 'f', 1, '-', '1', 2, '\\', + 'N', 1, '0', 2, '0', 1, '8', 2, 'a', 'b', 'c' }; TestLazyPrimitive.initLazyObject(b, data, 0, data.length); - - assertEquals(new Text("def"), ((LazyString)b.getMapValueElement(new IntWritable(2))).getWritableObject()); + + assertEquals(new Text("def"), ((LazyString) b + .getMapValueElement(new IntWritable(2))).getWritableObject()); assertNull(b.getMapValueElement(new IntWritable(-1))); - assertEquals(new Text("0"), ((LazyString)b.getMapValueElement(new IntWritable(0))).getWritableObject()); - assertEquals(new Text("abc"), ((LazyString)b.getMapValueElement(new IntWritable(8))).getWritableObject()); + assertEquals(new Text("0"), ((LazyString) b + .getMapValueElement(new IntWritable(0))).getWritableObject()); + assertEquals(new Text("abc"), ((LazyString) b + .getMapValueElement(new IntWritable(8))).getWritableObject()); assertNull(b.getMapValueElement(new IntWritable(12345))); - + assertEquals("{2:'def',-1:null,0:'0',8:'abc'}".replace('\'', '\"'), SerDeUtils.getJSONString(b, oi)); } - + { // Map of String to String Text nullSequence = new Text("\\N"); ObjectInspector oi = LazyFactory.createLazyObjectInspector( - TypeInfoUtils.getTypeInfosFromTypeString("map").get(0), - new byte[]{(byte)'#', (byte)'\t'}, 0, nullSequence, false, (byte)0); - LazyMap b = (LazyMap)LazyFactory.createLazyObject(oi); - byte[] data = new byte[]{'2', '\t', 'd', '\t', 'f', '#', '2', '\t', 'd', '#', '-', '1', '#', '0', '\t', '0', '#', '8', '\t', 'a', 'b', 'c'}; + TypeInfoUtils.getTypeInfosFromTypeString("map").get( + 0), new byte[] { (byte) '#', (byte) '\t' }, 0, nullSequence, + false, (byte) 0); + LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi); + byte[] data = new byte[] { '2', '\t', 'd', '\t', 'f', '#', '2', '\t', + 'd', '#', '-', '1', '#', '0', '\t', '0', '#', '8', '\t', 'a', 'b', + 'c' }; TestLazyPrimitive.initLazyObject(b, data, 0, data.length); - - assertEquals(new Text("d\tf"), ((LazyString)b.getMapValueElement(new Text("2"))).getWritableObject()); + + assertEquals(new Text("d\tf"), ((LazyString) b + .getMapValueElement(new Text("2"))).getWritableObject()); assertNull(b.getMapValueElement(new Text("-1"))); - assertEquals(new Text("0"), ((LazyString)b.getMapValueElement(new Text("0"))).getWritableObject()); - assertEquals(new Text("abc"), ((LazyString)b.getMapValueElement(new Text("8"))).getWritableObject()); + assertEquals(new Text("0"), ((LazyString) b + .getMapValueElement(new Text("0"))).getWritableObject()); + assertEquals(new Text("abc"), ((LazyString) b + .getMapValueElement(new Text("8"))).getWritableObject()); assertNull(b.getMapValueElement(new Text("-"))); - - assertEquals("{'2':'d\\tf','2':'d','-1':null,'0':'0','8':'abc'}".replace('\'', '\"'), - SerDeUtils.getJSONString(b, oi)); + + assertEquals("{'2':'d\\tf','2':'d','-1':null,'0':'0','8':'abc'}" + .replace('\'', '\"'), SerDeUtils.getJSONString(b, oi)); } - + } catch (Throwable e) { e.printStackTrace(); throw e; } } - + /** * Test the LazyStruct class. */ public void testLazyStruct() throws Throwable { try { { - ArrayList fieldTypeInfos = - TypeInfoUtils.getTypeInfosFromTypeString("int,array,map,string"); - List fieldNames = Arrays.asList(new String[]{"a", "b", "c", "d"}); + ArrayList fieldTypeInfos = TypeInfoUtils + .getTypeInfosFromTypeString("int,array,map,string"); + List fieldNames = Arrays.asList(new String[] { "a", "b", "c", + "d" }); Text nullSequence = new Text("\\N"); - + ObjectInspector oi = LazyFactory.createLazyStructInspector(fieldNames, - fieldTypeInfos, new byte[] {' ', ':', '='}, nullSequence, false, false, (byte)0); - LazyStruct o = (LazyStruct)LazyFactory.createLazyObject(oi); - + fieldTypeInfos, new byte[] { ' ', ':', '=' }, nullSequence, false, + false, (byte) 0); + LazyStruct o = (LazyStruct) LazyFactory.createLazyObject(oi); + Text data; - + data = new Text("123 a:b:c d=e:f=g hi"); - TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data.getLength()); - assertEquals("{'a':123,'b':['a','b','c'],'c':{'d':'e','f':'g'},'d':'hi'}".replace("'", "\""), - SerDeUtils.getJSONString(o, oi)); + TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data + .getLength()); + assertEquals( + "{'a':123,'b':['a','b','c'],'c':{'d':'e','f':'g'},'d':'hi'}" + .replace("'", "\""), SerDeUtils.getJSONString(o, oi)); data = new Text("123 \\N d=e:f=g \\N"); - TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data.getLength()); - assertEquals("{'a':123,'b':null,'c':{'d':'e','f':'g'},'d':null}".replace("'", "\""), - SerDeUtils.getJSONString(o, oi)); + TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data + .getLength()); + assertEquals("{'a':123,'b':null,'c':{'d':'e','f':'g'},'d':null}" + .replace("'", "\""), SerDeUtils.getJSONString(o, oi)); data = new Text("\\N a d=\\N:f=g:h no tail"); - TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data.getLength()); - assertEquals("{'a':null,'b':['a'],'c':{'d':null,'f':'g','h':null},'d':'no'}".replace("'", "\""), - SerDeUtils.getJSONString(o, oi)); + TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data + .getLength()); + assertEquals( + "{'a':null,'b':['a'],'c':{'d':null,'f':'g','h':null},'d':'no'}" + .replace("'", "\""), SerDeUtils.getJSONString(o, oi)); data = new Text("\\N :a:: \\N no tail"); - TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data.getLength()); - assertEquals("{'a':null,'b':['','a','',''],'c':null,'d':'no'}".replace("'", "\""), - SerDeUtils.getJSONString(o, oi)); + TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data + .getLength()); + assertEquals("{'a':null,'b':['','a','',''],'c':null,'d':'no'}".replace( + "'", "\""), SerDeUtils.getJSONString(o, oi)); data = new Text("123 "); - TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data.getLength()); + TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data + .getLength()); assertEquals("{'a':123,'b':[],'c':{},'d':''}".replace("'", "\""), SerDeUtils.getJSONString(o, oi)); data = new Text(": : : :"); - TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data.getLength()); - assertEquals("{'a':null,'b':['',''],'c':{'':null,'':null},'d':':'}".replace("'", "\""), - SerDeUtils.getJSONString(o, oi)); + TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data + .getLength()); + assertEquals("{'a':null,'b':['',''],'c':{'':null,'':null},'d':':'}" + .replace("'", "\""), SerDeUtils.getJSONString(o, oi)); data = new Text("= = = ="); - TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data.getLength()); - assertEquals("{'a':null,'b':['='],'c':{'':''},'d':'='}".replace("'", "\""), - SerDeUtils.getJSONString(o, oi)); - + TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data + .getLength()); + assertEquals("{'a':null,'b':['='],'c':{'':''},'d':'='}".replace("'", + "\""), SerDeUtils.getJSONString(o, oi)); + // test LastColumnTakesRest - oi = LazyFactory.createLazyStructInspector(Arrays.asList(new String[]{"a","b","c","d"}), - fieldTypeInfos, new byte[] {' ', ':', '='}, nullSequence, true, false, (byte)0); - o = (LazyStruct)LazyFactory.createLazyObject(oi); + oi = LazyFactory.createLazyStructInspector(Arrays.asList(new String[] { + "a", "b", "c", "d" }), fieldTypeInfos, + new byte[] { ' ', ':', '=' }, nullSequence, true, false, (byte) 0); + o = (LazyStruct) LazyFactory.createLazyObject(oi); data = new Text("\\N a d=\\N:f=g:h has tail"); - TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data.getLength()); - assertEquals("{'a':null,'b':['a'],'c':{'d':null,'f':'g','h':null},'d':'has tail'}".replace("'", "\""), - SerDeUtils.getJSONString(o, oi)); + TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data + .getLength()); + assertEquals( + "{'a':null,'b':['a'],'c':{'d':null,'f':'g','h':null},'d':'has tail'}" + .replace("'", "\""), SerDeUtils.getJSONString(o, oi)); } } catch (Throwable e) { e.printStackTrace(); throw e; } } - + } Index: serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java (working copy) @@ -17,110 +17,112 @@ */ package org.apache.hadoop.hive.serde2.lazy; +import junit.framework.TestCase; import org.apache.hadoop.hive.serde2.ByteStream; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; -import junit.framework.TestCase; - public class TestLazyPrimitive extends TestCase { /** - * Initialize the LazyObject with the parameters, wrapping the byte[] automatically. + * Initialize the LazyObject with the parameters, wrapping the byte[] + * automatically. */ - public static void initLazyObject(LazyObject lo, byte[] data, int start, int length) { + public static void initLazyObject(LazyObject lo, byte[] data, int start, + int length) { ByteArrayRef b = new ByteArrayRef(); b.setData(data); - lo.init(b, start, length); + lo.init(b, start, length); } + /** * Test the LazyByte class. */ public void testLazyByte() throws Throwable { try { - LazyByte b = new LazyByte(LazyPrimitiveObjectInspectorFactory.lazyByteObjectInspector); - initLazyObject(b,new byte[]{'0'}, 0, 0); + LazyByte b = new LazyByte( + LazyPrimitiveObjectInspectorFactory.lazyByteObjectInspector); + initLazyObject(b, new byte[] { '0' }, 0, 0); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'0'}, 0, 1); - assertEquals(new ByteWritable((byte)0), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '0'}, 0, 2); - assertEquals(new ByteWritable((byte)0), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '0'}, 0, 2); - assertEquals(new ByteWritable((byte)0), b.getWritableObject()); - initLazyObject(b,new byte[]{'a', '1', 'b'}, 1, 1); - assertEquals(new ByteWritable((byte)1), b.getWritableObject()); - initLazyObject(b,new byte[]{'a', '-', '1'}, 1, 2); - assertEquals(new ByteWritable((byte)-1), b.getWritableObject()); - initLazyObject(b,new byte[]{'a', '+', '1'}, 1, 2); - assertEquals(new ByteWritable((byte)1), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '1', '2', '8'}, 0, 4); - assertEquals(new ByteWritable((byte)-128), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '1', '2', '7'}, 0, 4); - assertEquals(new ByteWritable((byte)127), b.getWritableObject()); - - initLazyObject(b,new byte[]{'a', '1', 'b'}, 1, 2); + initLazyObject(b, new byte[] { '0' }, 0, 1); + assertEquals(new ByteWritable((byte) 0), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '0' }, 0, 2); + assertEquals(new ByteWritable((byte) 0), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '0' }, 0, 2); + assertEquals(new ByteWritable((byte) 0), b.getWritableObject()); + initLazyObject(b, new byte[] { 'a', '1', 'b' }, 1, 1); + assertEquals(new ByteWritable((byte) 1), b.getWritableObject()); + initLazyObject(b, new byte[] { 'a', '-', '1' }, 1, 2); + assertEquals(new ByteWritable((byte) -1), b.getWritableObject()); + initLazyObject(b, new byte[] { 'a', '+', '1' }, 1, 2); + assertEquals(new ByteWritable((byte) 1), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '1', '2', '8' }, 0, 4); + assertEquals(new ByteWritable((byte) -128), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '1', '2', '7' }, 0, 4); + assertEquals(new ByteWritable((byte) 127), b.getWritableObject()); + + initLazyObject(b, new byte[] { 'a', '1', 'b' }, 1, 2); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '1', '2', '8'}, 0, 4); + initLazyObject(b, new byte[] { '+', '1', '2', '8' }, 0, 4); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '1', '2', '9'}, 0, 4); + initLazyObject(b, new byte[] { '-', '1', '2', '9' }, 0, 4); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '1', '2', '3'}, 0, 1); + initLazyObject(b, new byte[] { '-', '1', '2', '3' }, 0, 1); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '1', '2', '3'}, 0, 1); + initLazyObject(b, new byte[] { '+', '1', '2', '3' }, 0, 1); assertNull(b.getWritableObject()); - + } catch (Throwable e) { e.printStackTrace(); throw e; } } - + /** * Test the LazyShort class. */ public void testLazyShort() throws Throwable { try { - LazyShort b = new LazyShort(LazyPrimitiveObjectInspectorFactory.lazyShortObjectInspector); - initLazyObject(b,new byte[]{'0'}, 0, 0); + LazyShort b = new LazyShort( + LazyPrimitiveObjectInspectorFactory.lazyShortObjectInspector); + initLazyObject(b, new byte[] { '0' }, 0, 0); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'0'}, 0, 1); - assertEquals(new ShortWritable((short)0), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '0'}, 0, 2); - assertEquals(new ShortWritable((short)0), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '0'}, 0, 2); - assertEquals(new ShortWritable((short)0), b.getWritableObject()); - initLazyObject(b,new byte[]{'a', '1', 'b'}, 1, 1); - assertEquals(new ShortWritable((short)1), b.getWritableObject()); - initLazyObject(b,new byte[]{'a', '-', '1'}, 1, 2); - assertEquals(new ShortWritable((short)-1), b.getWritableObject()); - initLazyObject(b,new byte[]{'a', '+', '1'}, 1, 2); - assertEquals(new ShortWritable((short)1), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '1', '2', '8'}, 0, 4); - assertEquals(new ShortWritable((short)-128), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '1', '2', '7'}, 0, 4); - assertEquals(new ShortWritable((short)127), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '3', '2', '7', '6', '8'}, 0, 6); - assertEquals(new ShortWritable((short)-32768), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '3', '2', '7', '6', '7'}, 0, 6); - assertEquals(new ShortWritable((short)32767), b.getWritableObject()); + initLazyObject(b, new byte[] { '0' }, 0, 1); + assertEquals(new ShortWritable((short) 0), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '0' }, 0, 2); + assertEquals(new ShortWritable((short) 0), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '0' }, 0, 2); + assertEquals(new ShortWritable((short) 0), b.getWritableObject()); + initLazyObject(b, new byte[] { 'a', '1', 'b' }, 1, 1); + assertEquals(new ShortWritable((short) 1), b.getWritableObject()); + initLazyObject(b, new byte[] { 'a', '-', '1' }, 1, 2); + assertEquals(new ShortWritable((short) -1), b.getWritableObject()); + initLazyObject(b, new byte[] { 'a', '+', '1' }, 1, 2); + assertEquals(new ShortWritable((short) 1), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '1', '2', '8' }, 0, 4); + assertEquals(new ShortWritable((short) -128), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '1', '2', '7' }, 0, 4); + assertEquals(new ShortWritable((short) 127), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '3', '2', '7', '6', '8' }, 0, 6); + assertEquals(new ShortWritable((short) -32768), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '3', '2', '7', '6', '7' }, 0, 6); + assertEquals(new ShortWritable((short) 32767), b.getWritableObject()); - initLazyObject(b,new byte[]{'a', '1', 'b'}, 1, 2); + initLazyObject(b, new byte[] { 'a', '1', 'b' }, 1, 2); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '3', '2', '7', '6', '9'}, 0, 6); + initLazyObject(b, new byte[] { '-', '3', '2', '7', '6', '9' }, 0, 6); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '3', '2', '7', '6', '8'}, 0, 6); + initLazyObject(b, new byte[] { '+', '3', '2', '7', '6', '8' }, 0, 6); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '1', '2', '3'}, 0, 1); + initLazyObject(b, new byte[] { '-', '1', '2', '3' }, 0, 1); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '1', '2', '3'}, 0, 1); + initLazyObject(b, new byte[] { '+', '1', '2', '3' }, 0, 1); assertNull(b.getWritableObject()); } catch (Throwable e) { @@ -128,50 +130,54 @@ throw e; } } - /** * Test the LazyInteger class. */ public void testLazyInteger() throws Throwable { try { - LazyInteger b = new LazyInteger(LazyPrimitiveObjectInspectorFactory.lazyIntObjectInspector); - initLazyObject(b,new byte[]{'0'}, 0, 0); + LazyInteger b = new LazyInteger( + LazyPrimitiveObjectInspectorFactory.lazyIntObjectInspector); + initLazyObject(b, new byte[] { '0' }, 0, 0); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'0'}, 0, 1); - assertEquals(new IntWritable((int)0), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '0'}, 0, 2); - assertEquals(new IntWritable((int)0), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '0'}, 0, 2); - assertEquals(new IntWritable((int)0), b.getWritableObject()); - initLazyObject(b,new byte[]{'a', '1', 'b'}, 1, 1); - assertEquals(new IntWritable((int)1), b.getWritableObject()); - initLazyObject(b,new byte[]{'a', '-', '1'}, 1, 2); - assertEquals(new IntWritable((int)-1), b.getWritableObject()); - initLazyObject(b,new byte[]{'a', '+', '1'}, 1, 2); - assertEquals(new IntWritable((int)1), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '1', '2', '8'}, 0, 4); - assertEquals(new IntWritable((int)-128), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '1', '2', '7'}, 0, 4); - assertEquals(new IntWritable((int)127), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '3', '2', '7', '6', '8'}, 0, 6); - assertEquals(new IntWritable((int)-32768), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '3', '2', '7', '6', '7'}, 0, 6); - assertEquals(new IntWritable((int)32767), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '2', '1', '4', '7', '4', '8', '3', '6', '4', '8'}, 0, 11); - assertEquals(new IntWritable((int)-2147483648), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '2', '1', '4', '7', '4', '8', '3', '6', '4', '7'}, 0, 11); - assertEquals(new IntWritable((int)2147483647), b.getWritableObject()); + initLazyObject(b, new byte[] { '0' }, 0, 1); + assertEquals(new IntWritable(0), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '0' }, 0, 2); + assertEquals(new IntWritable(0), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '0' }, 0, 2); + assertEquals(new IntWritable(0), b.getWritableObject()); + initLazyObject(b, new byte[] { 'a', '1', 'b' }, 1, 1); + assertEquals(new IntWritable(1), b.getWritableObject()); + initLazyObject(b, new byte[] { 'a', '-', '1' }, 1, 2); + assertEquals(new IntWritable(-1), b.getWritableObject()); + initLazyObject(b, new byte[] { 'a', '+', '1' }, 1, 2); + assertEquals(new IntWritable(1), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '1', '2', '8' }, 0, 4); + assertEquals(new IntWritable(-128), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '1', '2', '7' }, 0, 4); + assertEquals(new IntWritable(127), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '3', '2', '7', '6', '8' }, 0, 6); + assertEquals(new IntWritable(-32768), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '3', '2', '7', '6', '7' }, 0, 6); + assertEquals(new IntWritable(32767), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '2', '1', '4', '7', '4', '8', '3', + '6', '4', '8' }, 0, 11); + assertEquals(new IntWritable(-2147483648), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '2', '1', '4', '7', '4', '8', '3', + '6', '4', '7' }, 0, 11); + assertEquals(new IntWritable(2147483647), b.getWritableObject()); - initLazyObject(b,new byte[]{'a', '1', 'b'}, 1, 2); + initLazyObject(b, new byte[] { 'a', '1', 'b' }, 1, 2); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '2', '1', '4', '7', '4', '8', '3', '6', '4', '9'}, 0, 11); + initLazyObject(b, new byte[] { '-', '2', '1', '4', '7', '4', '8', '3', + '6', '4', '9' }, 0, 11); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '2', '1', '4', '7', '4', '8', '3', '6', '4', '8'}, 0, 11); + initLazyObject(b, new byte[] { '+', '2', '1', '4', '7', '4', '8', '3', + '6', '4', '8' }, 0, 11); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '1', '2', '3'}, 0, 1); + initLazyObject(b, new byte[] { '-', '1', '2', '3' }, 0, 1); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '1', '2', '3'}, 0, 1); + initLazyObject(b, new byte[] { '+', '1', '2', '3' }, 0, 1); assertNull(b.getWritableObject()); } catch (Throwable e) { @@ -179,58 +185,62 @@ throw e; } } - - + /** * Test the LazyLong class. */ public void testLazyLong() throws Throwable { try { - LazyLong b = new LazyLong(LazyPrimitiveObjectInspectorFactory.lazyLongObjectInspector); - initLazyObject(b,new byte[]{'0'}, 0, 0); + LazyLong b = new LazyLong( + LazyPrimitiveObjectInspectorFactory.lazyLongObjectInspector); + initLazyObject(b, new byte[] { '0' }, 0, 0); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'0'}, 0, 1); - assertEquals(new LongWritable((long)0), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '0'}, 0, 2); - assertEquals(new LongWritable((long)0), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '0'}, 0, 2); - assertEquals(new LongWritable((long)0), b.getWritableObject()); - initLazyObject(b,new byte[]{'a', '1', 'b'}, 1, 1); - assertEquals(new LongWritable((long)1), b.getWritableObject()); - initLazyObject(b,new byte[]{'a', '-', '1'}, 1, 2); - assertEquals(new LongWritable((long)-1), b.getWritableObject()); - initLazyObject(b,new byte[]{'a', '+', '1'}, 1, 2); - assertEquals(new LongWritable((long)1), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '1', '2', '8'}, 0, 4); - assertEquals(new LongWritable((long)-128), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '1', '2', '7'}, 0, 4); - assertEquals(new LongWritable((long)127), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '3', '2', '7', '6', '8'}, 0, 6); - assertEquals(new LongWritable((long)-32768), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '3', '2', '7', '6', '7'}, 0, 6); - assertEquals(new LongWritable((long)32767), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '2', '1', '4', '7', '4', '8', '3', '6', '4', '8'}, 0, 11); - assertEquals(new LongWritable((long)-2147483648), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '2', '1', '4', '7', '4', '8', '3', '6', '4', '7'}, 0, 11); - assertEquals(new LongWritable((long)2147483647), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '9', '2', '2', '3', '3', '7', '2', '0', '3', '6', '8', '5', - '4', '7', '7', '5', '8', '0', '8'}, 0, 20); - assertEquals(new LongWritable((long)-9223372036854775808L), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '9', '2', '2', '3', '3', '7', '2', '0', '3', '6', '8', '5', - '4', '7', '7', '5', '8', '0', '7'}, 0, 20); - assertEquals(new LongWritable((long)9223372036854775807L), b.getWritableObject()); + initLazyObject(b, new byte[] { '0' }, 0, 1); + assertEquals(new LongWritable(0), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '0' }, 0, 2); + assertEquals(new LongWritable(0), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '0' }, 0, 2); + assertEquals(new LongWritable(0), b.getWritableObject()); + initLazyObject(b, new byte[] { 'a', '1', 'b' }, 1, 1); + assertEquals(new LongWritable(1), b.getWritableObject()); + initLazyObject(b, new byte[] { 'a', '-', '1' }, 1, 2); + assertEquals(new LongWritable(-1), b.getWritableObject()); + initLazyObject(b, new byte[] { 'a', '+', '1' }, 1, 2); + assertEquals(new LongWritable(1), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '1', '2', '8' }, 0, 4); + assertEquals(new LongWritable(-128), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '1', '2', '7' }, 0, 4); + assertEquals(new LongWritable(127), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '3', '2', '7', '6', '8' }, 0, 6); + assertEquals(new LongWritable(-32768), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '3', '2', '7', '6', '7' }, 0, 6); + assertEquals(new LongWritable(32767), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '2', '1', '4', '7', '4', '8', '3', + '6', '4', '8' }, 0, 11); + assertEquals(new LongWritable(-2147483648), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '2', '1', '4', '7', '4', '8', '3', + '6', '4', '7' }, 0, 11); + assertEquals(new LongWritable(2147483647), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '9', '2', '2', '3', '3', '7', '2', + '0', '3', '6', '8', '5', '4', '7', '7', '5', '8', '0', '8' }, 0, 20); + assertEquals(new LongWritable(-9223372036854775808L), b + .getWritableObject()); + initLazyObject(b, new byte[] { '+', '9', '2', '2', '3', '3', '7', '2', + '0', '3', '6', '8', '5', '4', '7', '7', '5', '8', '0', '7' }, 0, 20); + assertEquals(new LongWritable(9223372036854775807L), b + .getWritableObject()); - initLazyObject(b,new byte[]{'a', '1', 'b'}, 1, 2); + initLazyObject(b, new byte[] { 'a', '1', 'b' }, 1, 2); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '9', '2', '2', '3', '3', '7', '2', '0', '3', '6', '8', '5', - '4', '7', '7', '5', '8', '0', '9'}, 0, 20); + initLazyObject(b, new byte[] { '-', '9', '2', '2', '3', '3', '7', '2', + '0', '3', '6', '8', '5', '4', '7', '7', '5', '8', '0', '9' }, 0, 20); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '9', '2', '2', '3', '3', '7', '2', '0', '3', '6', '8', '5', - '4', '7', '7', '5', '8', '0', '8'}, 0, 20); + initLazyObject(b, new byte[] { '+', '9', '2', '2', '3', '3', '7', '2', + '0', '3', '6', '8', '5', '4', '7', '7', '5', '8', '0', '8' }, 0, 20); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '1', '2', '3'}, 0, 1); + initLazyObject(b, new byte[] { '-', '1', '2', '3' }, 0, 1); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '1', '2', '3'}, 0, 1); + initLazyObject(b, new byte[] { '+', '1', '2', '3' }, 0, 1); assertNull(b.getWritableObject()); } catch (Throwable e) { @@ -244,150 +254,163 @@ */ public void testLazyDouble() throws Throwable { try { - LazyDouble b = new LazyDouble(LazyPrimitiveObjectInspectorFactory.lazyDoubleObjectInspector); - initLazyObject(b,new byte[]{'0'}, 0, 0); + LazyDouble b = new LazyDouble( + LazyPrimitiveObjectInspectorFactory.lazyDoubleObjectInspector); + initLazyObject(b, new byte[] { '0' }, 0, 0); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'0'}, 0, 1); - assertEquals(new DoubleWritable((double)0), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '0'}, 0, 2); - assertEquals(new DoubleWritable((double)0), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '0'}, 0, 2); - assertEquals(new DoubleWritable((double)-0.0), b.getWritableObject()); - initLazyObject(b,new byte[]{'a', '1', 'b'}, 1, 1); - assertEquals(new DoubleWritable((double)1), b.getWritableObject()); - initLazyObject(b,new byte[]{'a', '-', '1'}, 1, 2); - assertEquals(new DoubleWritable((double)-1), b.getWritableObject()); - initLazyObject(b,new byte[]{'a', '+', '1'}, 1, 2); - assertEquals(new DoubleWritable((double)1), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '1', '2', '8'}, 0, 4); - assertEquals(new DoubleWritable((double)-128), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '1', '2', '7'}, 0, 4); - assertEquals(new DoubleWritable((double)127), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '3', '2', '7', '6', '8'}, 0, 6); - assertEquals(new DoubleWritable((double)-32768), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '3', '2', '7', '6', '7'}, 0, 6); - assertEquals(new DoubleWritable((double)32767), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '2', '1', '4', '7', '4', '8', '3', '6', '4', '8'}, 0, 11); - assertEquals(new DoubleWritable((double)-2147483648), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '2', '1', '4', '7', '4', '8', '3', '6', '4', '7'}, 0, 11); - assertEquals(new DoubleWritable((double)2147483647), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '9', '2', '2', '3', '3', '7', '2', '0', '3', '6', '8', '5', - '4', '7', '7', '5', '8', '0', '8'}, 0, 20); - assertEquals(new DoubleWritable((double)-9223372036854775808L), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '9', '2', '2', '3', '3', '7', '2', '0', '3', '6', '8', '5', - '4', '7', '7', '5', '8', '0', '7'}, 0, 20); - assertEquals(new DoubleWritable((long)9223372036854775807L), b.getWritableObject()); + initLazyObject(b, new byte[] { '0' }, 0, 1); + assertEquals(new DoubleWritable(0), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '0' }, 0, 2); + assertEquals(new DoubleWritable(0), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '0' }, 0, 2); + assertEquals(new DoubleWritable(-0.0), b.getWritableObject()); + initLazyObject(b, new byte[] { 'a', '1', 'b' }, 1, 1); + assertEquals(new DoubleWritable(1), b.getWritableObject()); + initLazyObject(b, new byte[] { 'a', '-', '1' }, 1, 2); + assertEquals(new DoubleWritable(-1), b.getWritableObject()); + initLazyObject(b, new byte[] { 'a', '+', '1' }, 1, 2); + assertEquals(new DoubleWritable(1), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '1', '2', '8' }, 0, 4); + assertEquals(new DoubleWritable(-128), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '1', '2', '7' }, 0, 4); + assertEquals(new DoubleWritable(127), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '3', '2', '7', '6', '8' }, 0, 6); + assertEquals(new DoubleWritable(-32768), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '3', '2', '7', '6', '7' }, 0, 6); + assertEquals(new DoubleWritable(32767), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '2', '1', '4', '7', '4', '8', '3', + '6', '4', '8' }, 0, 11); + assertEquals(new DoubleWritable(-2147483648), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '2', '1', '4', '7', '4', '8', '3', + '6', '4', '7' }, 0, 11); + assertEquals(new DoubleWritable(2147483647), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '9', '2', '2', '3', '3', '7', '2', + '0', '3', '6', '8', '5', '4', '7', '7', '5', '8', '0', '8' }, 0, 20); + assertEquals(new DoubleWritable(-9223372036854775808L), b + .getWritableObject()); + initLazyObject(b, new byte[] { '+', '9', '2', '2', '3', '3', '7', '2', + '0', '3', '6', '8', '5', '4', '7', '7', '5', '8', '0', '7' }, 0, 20); + assertEquals(new DoubleWritable(9223372036854775807L), b + .getWritableObject()); - initLazyObject(b,new byte[]{'-', '3', '.', '7', '6', '8'}, 0, 6); - assertEquals(new DoubleWritable((double)-3.768), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '3', '.', '7', '6', '7'}, 0, 6); - assertEquals(new DoubleWritable((double)3.767), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '2', '.', '4', '7', '4', '8', '3', '6', 'e', '8'}, 0, 11); - assertEquals(new DoubleWritable((double)-2.474836e8), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '2', '.', '4', '7', '4', '8', '3', 'E', '-', '7'}, 0, 11); - assertEquals(new DoubleWritable((double)2.47483E-7), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '.', '4', '7', '4', '8', '3', '6', 'e', '8'}, 0, 10); - assertEquals(new DoubleWritable((double)-.474836e8), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '.', '4', '7', '4', '8', '3', 'E', '-', '7'}, 0, 10); - assertEquals(new DoubleWritable((double).47483E-7), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '2', '1', '4', '7', '4', '8', '3', '6', '4', '.'}, 0, 11); - assertEquals(new DoubleWritable((double)-214748364.), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '2', '1', '4', '7', '4', '8', '3', '6', '4', '.'}, 0, 11); - assertEquals(new DoubleWritable((double)+214748364.), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '3', '.', '7', '6', '8' }, 0, 6); + assertEquals(new DoubleWritable(-3.768), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '3', '.', '7', '6', '7' }, 0, 6); + assertEquals(new DoubleWritable(3.767), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '2', '.', '4', '7', '4', '8', '3', + '6', 'e', '8' }, 0, 11); + assertEquals(new DoubleWritable(-2.474836e8), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '2', '.', '4', '7', '4', '8', '3', + 'E', '-', '7' }, 0, 11); + assertEquals(new DoubleWritable(2.47483E-7), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '.', '4', '7', '4', '8', '3', '6', + 'e', '8' }, 0, 10); + assertEquals(new DoubleWritable(-.474836e8), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '.', '4', '7', '4', '8', '3', 'E', + '-', '7' }, 0, 10); + assertEquals(new DoubleWritable(.47483E-7), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '2', '1', '4', '7', '4', '8', '3', + '6', '4', '.' }, 0, 11); + assertEquals(new DoubleWritable(-214748364.), b.getWritableObject()); + initLazyObject(b, new byte[] { '+', '2', '1', '4', '7', '4', '8', '3', + '6', '4', '.' }, 0, 11); + assertEquals(new DoubleWritable(+214748364.), b.getWritableObject()); - initLazyObject(b,new byte[]{'.', '0'}, 0, 2); - assertEquals(new DoubleWritable((double).0), b.getWritableObject()); - initLazyObject(b,new byte[]{'0', '.'}, 0, 2); - assertEquals(new DoubleWritable((double)0.), b.getWritableObject()); - - initLazyObject(b,new byte[]{'a', '1', 'b'}, 1, 2); + initLazyObject(b, new byte[] { '.', '0' }, 0, 2); + assertEquals(new DoubleWritable(.0), b.getWritableObject()); + initLazyObject(b, new byte[] { '0', '.' }, 0, 2); + assertEquals(new DoubleWritable(0.), b.getWritableObject()); + + initLazyObject(b, new byte[] { 'a', '1', 'b' }, 1, 2); assertNull(b.getWritableObject()); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'.', '1', '2', '3'}, 0, 1); + initLazyObject(b, new byte[] { '.', '1', '2', '3' }, 0, 1); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '1', '2', '3'}, 0, 1); + initLazyObject(b, new byte[] { '-', '1', '2', '3' }, 0, 1); assertNull(b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '1', '2', '3'}, 0, 1); + initLazyObject(b, new byte[] { '+', '1', '2', '3' }, 0, 1); assertNull(b.getWritableObject()); - - initLazyObject(b,new byte[]{'-', '1', 'e', '3', '3', '3', '3', '3', '3'}, 0, 9); - assertEquals(new DoubleWritable(Double.NEGATIVE_INFINITY), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '1', 'e', '3', '3', '3', '3', '3', '3'}, 0, 9); - assertEquals(new DoubleWritable(Double.POSITIVE_INFINITY), b.getWritableObject()); - initLazyObject(b,new byte[]{'+', '1', 'e', '-', '3', '3', '3', '3', '3'}, 0, 8); - assertEquals(new DoubleWritable((double)0), b.getWritableObject()); - initLazyObject(b,new byte[]{'-', '1', 'e', '-', '3', '3', '3', '3', '3'}, 0, 8); - assertEquals(new DoubleWritable((double)-0.0), b.getWritableObject()); - + initLazyObject(b, new byte[] { '-', '1', 'e', '3', '3', '3', '3', '3', + '3' }, 0, 9); + assertEquals(new DoubleWritable(Double.NEGATIVE_INFINITY), b + .getWritableObject()); + initLazyObject(b, new byte[] { '+', '1', 'e', '3', '3', '3', '3', '3', + '3' }, 0, 9); + assertEquals(new DoubleWritable(Double.POSITIVE_INFINITY), b + .getWritableObject()); + + initLazyObject(b, new byte[] { '+', '1', 'e', '-', '3', '3', '3', '3', + '3' }, 0, 8); + assertEquals(new DoubleWritable(0), b.getWritableObject()); + initLazyObject(b, new byte[] { '-', '1', 'e', '-', '3', '3', '3', '3', + '3' }, 0, 8); + assertEquals(new DoubleWritable(-0.0), b.getWritableObject()); + } catch (Throwable e) { e.printStackTrace(); throw e; } } - + /** * Test the LazyString class. */ public void testLazyString() throws Throwable { try { LazyString b = new LazyString(LazyPrimitiveObjectInspectorFactory - .getLazyStringObjectInspector(false,(byte)0)); - initLazyObject(b,new byte[]{'0'}, 0, 0); + .getLazyStringObjectInspector(false, (byte) 0)); + initLazyObject(b, new byte[] { '0' }, 0, 0); assertEquals(new Text(""), b.getWritableObject()); - initLazyObject(b,new byte[]{'0'}, 0, 1); + initLazyObject(b, new byte[] { '0' }, 0, 1); assertEquals(new Text("0"), b.getWritableObject()); - initLazyObject(b,new byte[]{'0', '1', '2'}, 1, 1); + initLazyObject(b, new byte[] { '0', '1', '2' }, 1, 1); assertEquals(new Text("1"), b.getWritableObject()); - + } catch (Throwable e) { e.printStackTrace(); throw e; } } - + public void testLazyIntegerWrite() throws Throwable { try { ByteStream.Output out = new ByteStream.Output(); - - int[] tests = {0, -1, 1, -10, 10, -123, 123, - Integer.MIN_VALUE, Integer.MIN_VALUE + 1, - Integer.MAX_VALUE, Integer.MAX_VALUE - 1}; - for (int i=0; i fieldRefs = oi.getAllStructFieldRefs(); assertEquals(8, fieldRefs.size()); - + // Deserialize Object row = serDe.deserialize(t); for (int i = 0; i < fieldRefs.size(); i++) { Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i)); if (fieldData != null) { - fieldData = ((LazyPrimitive)fieldData).getWritableObject(); + fieldData = ((LazyPrimitive) fieldData).getWritableObject(); } assertEquals("Field " + i, expectedFieldsData[i], fieldData); } - // Serialize + // Serialize assertEquals(Text.class, serDe.getSerializedClass()); - Text serializedText = (Text)serDe.serialize(row, oi); + Text serializedText = (Text) serDe.serialize(row, oi); assertEquals("Serialized data", s, serializedText.toString()); } private Properties createProperties() { Properties tbl = new Properties(); - + // Set the configuration parameters tbl.setProperty(Constants.SERIALIZATION_FORMAT, "9"); tbl.setProperty("columns", @@ -100,7 +99,7 @@ tbl.setProperty(Constants.SERIALIZATION_NULL_FORMAT, "NULL"); return tbl; } - + /** * Test the LazySimpleSerDe class with LastColumnTakesRest option. */ @@ -112,16 +111,15 @@ Properties tbl = createProperties(); tbl.setProperty(Constants.SERIALIZATION_LAST_COLUMN_TAKES_REST, "true"); serDe.initialize(conf, tbl); - + // Data Text t = new Text("123\t456\t789\t1000\t5.3\thive and hadoop\t1.\ta\tb\t"); - String s = "123\t456\t789\t1000\t5.3\thive and hadoop\tNULL\ta\tb\t"; - Object[] expectedFieldsData = { new ByteWritable((byte)123), - new ShortWritable((short)456), new IntWritable(789), - new LongWritable(1000), new DoubleWritable(5.3), new Text("hive and hadoop"), null, - new Text("a\tb\t") - }; - + String s = "123\t456\t789\t1000\t5.3\thive and hadoop\tNULL\ta\tb\t"; + Object[] expectedFieldsData = { new ByteWritable((byte) 123), + new ShortWritable((short) 456), new IntWritable(789), + new LongWritable(1000), new DoubleWritable(5.3), + new Text("hive and hadoop"), null, new Text("a\tb\t") }; + // Test deserializeAndSerialize(serDe, t, s, expectedFieldsData); @@ -131,7 +129,6 @@ } } - /** * Test the LazySimpleSerDe class with extra columns. */ @@ -142,16 +139,15 @@ Configuration conf = new Configuration(); Properties tbl = createProperties(); serDe.initialize(conf, tbl); - + // Data Text t = new Text("123\t456\t789\t1000\t5.3\thive and hadoop\t1.\ta\tb\t"); - String s = "123\t456\t789\t1000\t5.3\thive and hadoop\tNULL\ta"; - Object[] expectedFieldsData = { new ByteWritable((byte)123), - new ShortWritable((short)456), new IntWritable(789), - new LongWritable(1000), new DoubleWritable(5.3), new Text("hive and hadoop"), null, - new Text("a") - }; - + String s = "123\t456\t789\t1000\t5.3\thive and hadoop\tNULL\ta"; + Object[] expectedFieldsData = { new ByteWritable((byte) 123), + new ShortWritable((short) 456), new IntWritable(789), + new LongWritable(1000), new DoubleWritable(5.3), + new Text("hive and hadoop"), null, new Text("a") }; + // Test deserializeAndSerialize(serDe, t, s, expectedFieldsData); @@ -161,7 +157,6 @@ } } - /** * Test the LazySimpleSerDe class with missing columns. */ @@ -172,16 +167,15 @@ Configuration conf = new Configuration(); Properties tbl = createProperties(); serDe.initialize(conf, tbl); - + // Data Text t = new Text("123\t456\t789\t1000\t5.3\t"); - String s = "123\t456\t789\t1000\t5.3\t\tNULL\tNULL"; - Object[] expectedFieldsData = { new ByteWritable((byte)123), - new ShortWritable((short)456), new IntWritable(789), + String s = "123\t456\t789\t1000\t5.3\t\tNULL\tNULL"; + Object[] expectedFieldsData = { new ByteWritable((byte) 123), + new ShortWritable((short) 456), new IntWritable(789), new LongWritable(1000), new DoubleWritable(5.3), new Text(""), null, - null - }; - + null }; + // Test deserializeAndSerialize(serDe, t, s, expectedFieldsData); @@ -190,5 +184,5 @@ throw e; } } - + } Index: serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/CreateSequenceFile.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/CreateSequenceFile.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/CreateSequenceFile.java (working copy) @@ -25,11 +25,13 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.serde2.ByteStream; +import org.apache.hadoop.hive.serde2.thrift.test.Complex; +import org.apache.hadoop.hive.serde2.thrift.test.IntString; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; - import org.apache.thrift.TBase; import org.apache.thrift.TException; import org.apache.thrift.protocol.TBinaryProtocol; @@ -37,10 +39,6 @@ import org.apache.thrift.protocol.TProtocolFactory; import org.apache.thrift.transport.TIOStreamTransport; -import org.apache.hadoop.hive.serde2.ByteStream; -import org.apache.hadoop.hive.serde2.thrift.test.Complex; -import org.apache.hadoop.hive.serde2.thrift.test.IntString; - public class CreateSequenceFile { public static void usage() { @@ -49,18 +47,19 @@ } public static class ThriftSerializer { - + ByteStream.Output bos; TProtocol outProtocol; - + public ThriftSerializer() { bos = new ByteStream.Output(); TIOStreamTransport outTransport = new TIOStreamTransport(bos); TProtocolFactory outFactory = new TBinaryProtocol.Factory(); outProtocol = outFactory.getProtocol(outTransport); } - + BytesWritable bw = new BytesWritable(); + public BytesWritable serialize(TBase base) throws TException { bos.reset(); base.write(outProtocol); @@ -68,15 +67,15 @@ return bw; } } - + public static void main(String[] args) throws Exception { // Read parameters int lines = 10; - List extraArgs = new ArrayList(); - for(int ai=0; ai extraArgs = new ArrayList(); + for (int ai = 0; ai < args.length; ai++) { if (args[ai].equals("-line") && ai + 1 < args.length) { - lines = Integer.parseInt(args[ai+1]); + lines = Integer.parseInt(args[ai + 1]); ai++; } else { extraArgs.add(args[ai]); @@ -85,38 +84,39 @@ if (extraArgs.size() != 1) { usage(); } - + JobConf conf = new JobConf(CreateSequenceFile.class); - + ThriftSerializer serializer = new ThriftSerializer(); - + // Open files - SequenceFile.Writer writer = new SequenceFile.Writer(FileSystem.get(conf), conf, new Path(extraArgs.get(0)), - BytesWritable.class, BytesWritable.class); + SequenceFile.Writer writer = new SequenceFile.Writer(FileSystem.get(conf), + conf, new Path(extraArgs.get(0)), BytesWritable.class, + BytesWritable.class); // write to file BytesWritable key = new BytesWritable(); - + Random rand = new Random(20081215); - - for(int i=0; i alist = new ArrayList(); - alist.add(i); alist.add(i*2); alist.add(i*3); + alist.add(i); + alist.add(i * 2); + alist.add(i * 3); ArrayList slist = new ArrayList(); - slist.add("" + i*10); slist.add("" + i*100); slist.add("" + i*1000); + slist.add("" + i * 10); + slist.add("" + i * 100); + slist.add("" + i * 1000); ArrayList islist = new ArrayList(); - islist.add(new IntString(i*i, ""+ i*i*i, i)); - HashMap hash = new HashMap(); + islist.add(new IntString(i * i, "" + i * i * i, i)); + HashMap hash = new HashMap(); hash.put("key_" + i, "value_" + i); - - Complex complex = new Complex( rand.nextInt(), - "record_" + (new Integer(i)).toString(), - alist, - slist, - islist, - hash); + Complex complex = new Complex(rand.nextInt(), "record_" + + (new Integer(i)).toString(), alist, slist, islist, hash); + Writable value = serializer.serialize(complex); writer.append(key, value); } Index: serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java (working copy) @@ -23,6 +23,8 @@ import java.util.Properties; import java.util.Random; +import junit.framework.TestCase; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.SerDe; @@ -32,32 +34,30 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions; - -import junit.framework.TestCase; import org.apache.hadoop.io.BytesWritable; - public class TestBinarySortableSerDe extends TestCase { - public static HashMap makeHashMap(String... params) { - HashMap r = new HashMap(); - for(int i=0; i makeHashMap(String... params) { + HashMap r = new HashMap(); + for (int i = 0; i < params.length; i += 2) { + r.put(params[i], params[i + 1]); } return r; } public static String hexString(BytesWritable bytes) { StringBuilder sb = new StringBuilder(); - for (int i=0; i0) { - int compareResult = bytes[i-1].compareTo(bytes[i]); - if ( (compareResult<0 && !ascending) || (compareResult>0 && ascending) ) { - System.out.println("Test failed in " + (ascending ? "ascending" : "descending") - + " order with " + (i-1) + " and " + i); - System.out.println("serialized data [" + (i-1) + "] = " + hexString(bytes[i-1])); - System.out.println("serialized data [" + i + "] = " + hexString(bytes[i])); - System.out.println("deserialized data [" + (i-1) + " = " + - SerDeUtils.getJSONString(rows[i-1], rowOI)); - System.out.println("deserialized data [" + i + " = " + - SerDeUtils.getJSONString(rows[i], rowOI)); - fail("Sort order of serialized " + (i-1) + " and " + i + " are reversed!"); + if (i > 0) { + int compareResult = bytes[i - 1].compareTo(bytes[i]); + if ((compareResult < 0 && !ascending) + || (compareResult > 0 && ascending)) { + System.out.println("Test failed in " + + (ascending ? "ascending" : "descending") + " order with " + + (i - 1) + " and " + i); + System.out.println("serialized data [" + (i - 1) + "] = " + + hexString(bytes[i - 1])); + System.out.println("serialized data [" + i + "] = " + + hexString(bytes[i])); + System.out.println("deserialized data [" + (i - 1) + " = " + + SerDeUtils.getJSONString(rows[i - 1], rowOI)); + System.out.println("deserialized data [" + i + " = " + + SerDeUtils.getJSONString(rows[i], rowOI)); + fail("Sort order of serialized " + (i - 1) + " and " + i + + " are reversed!"); } } } // Try to deserialize Object[] deserialized = new Object[rows.length]; - for (int i=0; i 0 ? null : Byte.valueOf((byte)r.nextInt()); - t.myShort = randField > 1 ? null : Short.valueOf((short)r.nextInt()); + t.myByte = randField > 0 ? null : Byte.valueOf((byte) r.nextInt()); + t.myShort = randField > 1 ? null : Short.valueOf((short) r.nextInt()); t.myInt = randField > 2 ? null : Integer.valueOf(r.nextInt()); t.myLong = randField > 3 ? null : Long.valueOf(r.nextLong()); - t.myFloat = randField > 4 ? null : Float.valueOf(r.nextFloat() * 10 - 5); - t.myDouble = randField > 5 ? null : Double.valueOf(r.nextDouble() * 10 - 5); + t.myFloat = randField > 4 ? null : Float + .valueOf(r.nextFloat() * 10 - 5); + t.myDouble = randField > 5 ? null : Double + .valueOf(r.nextDouble() * 10 - 5); t.myString = randField > 6 ? null : getRandString(r); - t.myStruct = randField > 7 ? null : new MyTestInnerStruct(r.nextInt(5)-2, r.nextInt(5)-2); - t.myList = randField > 8 ? null: getRandIntegerArray(r); + t.myStruct = randField > 7 ? null : new MyTestInnerStruct( + r.nextInt(5) - 2, r.nextInt(5) - 2); + t.myList = randField > 8 ? null : getRandIntegerArray(r); rows[i] = t; } - StructObjectInspector rowOI = (StructObjectInspector)ObjectInspectorFactory - .getReflectionObjectInspector(MyTestClass.class, - ObjectInspectorOptions.JAVA); + StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory + .getReflectionObjectInspector(MyTestClass.class, + ObjectInspectorOptions.JAVA); sort(rows, rowOI); - + String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI); String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI); - - testBinarySortableSerDe(rows, rowOI, getSerDe(fieldNames, fieldTypes, "++++++++++"), true); - testBinarySortableSerDe(rows, rowOI, getSerDe(fieldNames, fieldTypes, "----------"), false); - + + testBinarySortableSerDe(rows, rowOI, getSerDe(fieldNames, fieldTypes, + "++++++++++"), true); + testBinarySortableSerDe(rows, rowOI, getSerDe(fieldNames, fieldTypes, + "----------"), false); + System.out.println("Test testTBinarySortableProtocol passed!"); } catch (Throwable e) { e.printStackTrace(); @@ -177,6 +193,4 @@ } } - - } Index: serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestInnerStruct.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestInnerStruct.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestInnerStruct.java (working copy) @@ -21,14 +21,13 @@ Integer int1; Integer int2; - + public MyTestInnerStruct() { } - + public MyTestInnerStruct(Integer int1, Integer int2) { this.int1 = int1; this.int2 = int2; } } - Index: serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java (working copy) @@ -29,21 +29,20 @@ String myString; MyTestInnerStruct myStruct; List myList; - - public MyTestClass() { + + public MyTestClass() { } - - public MyTestClass(Byte b, Short s, Integer i, Long l, - Float f, Double d, String st, MyTestInnerStruct is, - List li) { - this.myByte = b; - this.myShort = s; - this.myInt = i; - this.myLong = l; - this.myFloat = f; - this.myDouble = d; - this.myString = st; - this.myStruct = is; - this.myList = li; + + public MyTestClass(Byte b, Short s, Integer i, Long l, Float f, Double d, + String st, MyTestInnerStruct is, List li) { + myByte = b; + myShort = s; + myInt = i; + myLong = l; + myFloat = f; + myDouble = d; + myString = st; + myStruct = is; + myList = li; } } Index: serde/src/test/org/apache/hadoop/hive/serde2/TestTCTLSeparatedProtocol.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/TestTCTLSeparatedProtocol.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/TestTCTLSeparatedProtocol.java (working copy) @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.serde2; - import java.util.Properties; import junit.framework.TestCase; @@ -26,7 +25,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol; - import org.apache.thrift.protocol.TField; import org.apache.thrift.protocol.TList; import org.apache.thrift.protocol.TMap; @@ -44,15 +42,14 @@ String bar = "World!"; String key = "22"; - String value = "TheValue"; + String value = "TheValue"; String key2 = "24"; - String value2 = "TheValueAgain"; + String value2 = "TheValueAgain"; - byte columnSeparator [] = { 1 }; - byte elementSeparator [] = { 2 }; - byte kvSeparator [] = { 3 }; + byte columnSeparator[] = { 1 }; + byte elementSeparator[] = { 2 }; + byte kvSeparator[] = { 3 }; - trans.write(foo.getBytes(), 0, foo.getBytes().length); trans.write(columnSeparator, 0, 1); @@ -61,19 +58,17 @@ trans.write(bar.getBytes(), 0, bar.getBytes().length); trans.write(columnSeparator, 0, 1); - trans.write(key.getBytes(), 0, key.getBytes().length); + trans.write(key.getBytes(), 0, key.getBytes().length); trans.write(kvSeparator, 0, 1); trans.write(value.getBytes(), 0, value.getBytes().length); trans.write(elementSeparator, 0, 1); - trans.write(key2.getBytes(), 0, key2.getBytes().length); + trans.write(key2.getBytes(), 0, key2.getBytes().length); trans.write(kvSeparator, 0, 1); trans.write(value2.getBytes(), 0, value2.getBytes().length); - trans.flush(); - // use 3 as the row buffer size to force lots of re-buffering. TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 1024); prot.initialize(new Configuration(), new Properties()); @@ -113,7 +108,6 @@ prot.readStructEnd(); } - public void testWrites() throws Exception { TMemoryBuffer trans = new TMemoryBuffer(1024); TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 1024); @@ -152,15 +146,14 @@ prot.writeListEnd(); prot.writeFieldEnd(); - prot.writeFieldBegin(new TField()); prot.writeString("bye!"); prot.writeFieldEnd(); prot.writeStructEnd(); trans.flush(); - byte b[] = new byte[3*1024]; - int len = trans.read(b,0,b.length); + byte b[] = new byte[3 * 1024]; + int len = trans.read(b, 0, b.length); String test = new String(b, 0, len); String testRef = "100348.55234.22hello world!key1val1key2val2key3val3elem1elem2bye!"; @@ -175,7 +168,7 @@ prot = new TCTLSeparatedProtocol(trans, 10); prot.initialize(new Configuration(), new Properties()); - + // 100 is the start prot.readStructBegin(); prot.readFieldBegin(); @@ -265,11 +258,10 @@ trans.flush(); byte b[] = new byte[4096]; - int len = trans.read(b,0,b.length); + int len = trans.read(b, 0, b.length); - trans = new TMemoryBuffer(4096); - trans.write(b,0,len); + trans.write(b, 0, len); prot = new TCTLSeparatedProtocol(trans, 1024); prot.initialize(new Configuration(), schema); @@ -278,11 +270,10 @@ final String firstRead = prot.readString(); prot.readFieldEnd(); - testStr = testStr.replace("\"",""); + testStr = testStr.replace("\"", ""); assertEquals(testStr, firstRead); - // the 2 element list prot.readFieldBegin(); TList l = prot.readListBegin(); @@ -305,10 +296,10 @@ prot.readStructEnd(); } - /** - * Tests a sample apache log format. This is actually better done in general with a more TRegexLike protocol, but for this - * case, TCTLSeparatedProtocol can do it. + * Tests a sample apache log format. This is actually better done in general + * with a more TRegexLike protocol, but for this case, TCTLSeparatedProtocol + * can do it. */ public void test1ApacheLogFormat() throws Exception { final String sample = "127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326"; @@ -320,7 +311,8 @@ TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 4096); Properties schema = new Properties(); - // this is a hacky way of doing the quotes since it will match any 2 of these, so + // this is a hacky way of doing the quotes since it will match any 2 of + // these, so // "[ hello this is something to split [" would be considered to be quoted. schema.setProperty(Constants.QUOTE_CHAR, "(\"|\\[|\\])"); @@ -337,42 +329,42 @@ assertEquals("127.0.0.1", ip); - // identd + // identd prot.readFieldBegin(); final String identd = prot.readString(); prot.readFieldEnd(); assertNull(identd); - // user + // user prot.readFieldBegin(); final String user = prot.readString(); prot.readFieldEnd(); - assertEquals("frank",user); + assertEquals("frank", user); - // finishTime + // finishTime prot.readFieldBegin(); final String finishTime = prot.readString(); prot.readFieldEnd(); - assertEquals("10/Oct/2000:13:55:36 -0700",finishTime); + assertEquals("10/Oct/2000:13:55:36 -0700", finishTime); - // requestLine + // requestLine prot.readFieldBegin(); final String requestLine = prot.readString(); prot.readFieldEnd(); - assertEquals("GET /apache_pb.gif HTTP/1.0",requestLine); + assertEquals("GET /apache_pb.gif HTTP/1.0", requestLine); - // returncode + // returncode prot.readFieldBegin(); final int returnCode = prot.readI32(); prot.readFieldEnd(); assertEquals(200, returnCode); - // return size + // return size prot.readFieldBegin(); final int returnSize = prot.readI32(); prot.readFieldEnd(); @@ -382,8 +374,6 @@ prot.readStructEnd(); } - - public void testNulls() throws Exception { TMemoryBuffer trans = new TMemoryBuffer(1024); TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 10); @@ -420,9 +410,9 @@ prot.writeStructEnd(); - byte b[] = new byte[3*1024]; - int len = trans.read(b,0,b.length); - String written = new String(b,0,len); + byte b[] = new byte[3 * 1024]; + int len = trans.read(b, 0, b.length); + String written = new String(b, 0, len); String testRef = "\\N\\N100\\N\\N\\Nkey2\\N\\Nval3"; @@ -433,7 +423,7 @@ prot = new TCTLSeparatedProtocol(trans, 3); prot.initialize(new Configuration(), new Properties()); - + prot.readStructBegin(); prot.readFieldBegin(); @@ -445,7 +435,7 @@ prot.readFieldBegin(); ret = prot.readString(); prot.readFieldEnd(); - + assertNull(ret); prot.readFieldBegin(); @@ -454,13 +444,12 @@ assertTrue(ret1 == 100); - prot.readFieldBegin(); ret1 = prot.readI32(); prot.readFieldEnd(); prot.readFieldBegin(); - TMap map = prot.readMapBegin(); + TMap map = prot.readMapBegin(); assertTrue(map.size == 3); @@ -472,7 +461,7 @@ assertNull(prot.readString()); assertTrue(prot.readString().equals("val3")); - + prot.readMapEnd(); prot.readFieldEnd(); Index: serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java (working copy) @@ -25,6 +25,8 @@ import java.util.Properties; import java.util.Random; +import junit.framework.TestCase; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.SerDe; @@ -42,30 +44,34 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions; import org.apache.hadoop.io.BytesWritable; -import junit.framework.TestCase; +public class TestLazyBinarySerDe extends TestCase { -public class TestLazyBinarySerDe extends TestCase { - /** * Generate a random struct array - * @param r random number generator - * @return an struct array + * + * @param r + * random number generator + * @return an struct array */ static List getRandStructArray(Random r) { int length = r.nextInt(10); - ArrayList result = new ArrayList(length); - for(int i=0; i result = new ArrayList( + length); + for (int i = 0; i < length; i++) { MyTestInnerStruct ti = new MyTestInnerStruct(r.nextInt(), r.nextInt()); result.add(ti); } - return result; + return result; } - + /** * Initialize the LazyBinarySerDe - * @param fieldNames table field names - * @param fieldTypes table field types - * @return the initialized LazyBinarySerDe + * + * @param fieldNames + * table field names + * @param fieldTypes + * table field types + * @return the initialized LazyBinarySerDe * @throws Throwable */ private SerDe getSerDe(String fieldNames, String fieldTypes) throws Throwable { @@ -77,374 +83,418 @@ serde.initialize(new Configuration(), schema); return serde; } - + /** * Test the LazyBinarySerDe. - * @param rows array of structs to be serialized - * @param rowOI array of struct object inspectors - * @param serde the serde - * @throws Throwable + * + * @param rows + * array of structs to be serialized + * @param rowOI + * array of struct object inspectors + * @param serde + * the serde + * @throws Throwable */ - private void testLazyBinarySerDe(Object[] rows, - ObjectInspector rowOI, SerDe serde) throws Throwable{ - + private void testLazyBinarySerDe(Object[] rows, ObjectInspector rowOI, + SerDe serde) throws Throwable { + ObjectInspector serdeOI = serde.getObjectInspector(); // Try to serialize BytesWritable bytes[] = new BytesWritable[rows.length]; - for (int i=0; i entryinput: mp.entrySet()) { + + for (Map.Entry entryinput : mp.entrySet()) { boolean bEqual = false; - for (Map.Entry entryoutput: outputmp.entrySet()) { + for (Map.Entry entryoutput : outputmp.entrySet()) { // find the same key - if (0 == ObjectInspectorUtils.compare(entryoutput.getKey(), lazympkeyoi, entryinput.getKey(), inputmpkeyoi)) { - if(0 != ObjectInspectorUtils.compare(entryoutput.getValue(), lazympvalueoi, entryinput.getValue(), inputmpvalueoi)) { - assertEquals(entryoutput.getValue(), entryinput.getValue()); + if (0 == ObjectInspectorUtils.compare(entryoutput.getKey(), + lazympkeyoi, entryinput.getKey(), inputmpkeyoi)) { + if (0 != ObjectInspectorUtils.compare(entryoutput.getValue(), + lazympvalueoi, entryinput.getValue(), inputmpvalueoi)) { + assertEquals(entryoutput.getValue(), entryinput.getValue()); } else { bEqual = true; } break; - } + } } - if(!bEqual) - throw new RuntimeException("Could not find matched key in deserialized map : " + entryinput.getKey()); + if (!bEqual) { + throw new RuntimeException( + "Could not find matched key in deserialized map : " + + entryinput.getKey()); + } } } } - + /** * The test entrance function + * * @throws Throwable */ public void testLazyBinarySerDe() throws Throwable { try { System.out.println("Beginning Test TestLazyBinarySerDe:"); - + // generate the data int num = 1000; Random r = new Random(1234); MyTestClass rows[] = new MyTestClass[num]; - for (int i=0; i 0 ? null : Byte.valueOf((byte)r.nextInt()); - Short s = randField > 1 ? null : Short.valueOf((short)r.nextInt()); + Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt()); + Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt()); Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt()); - Long l = randField > 3 ? null : Long.valueOf(r.nextLong()); - Float f = randField > 4 ? null : Float.valueOf(r.nextFloat()); - Double d = randField > 5 ? null : Double.valueOf(r.nextDouble()); - String st = randField > 6 ? null : TestBinarySortableSerDe.getRandString(r); - MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r.nextInt(5)-2, r.nextInt(5)-2); - List li = randField > 8 ? null: TestBinarySortableSerDe.getRandIntegerArray(r); - MyTestClass t = new MyTestClass(b,s,n,l,f,d,st,is,li); + Long l = randField > 3 ? null : Long.valueOf(r.nextLong()); + Float f = randField > 4 ? null : Float.valueOf(r.nextFloat()); + Double d = randField > 5 ? null : Double.valueOf(r.nextDouble()); + String st = randField > 6 ? null : TestBinarySortableSerDe + .getRandString(r); + MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r + .nextInt(5) - 2, r.nextInt(5) - 2); + List li = randField > 8 ? null : TestBinarySortableSerDe + .getRandIntegerArray(r); + MyTestClass t = new MyTestClass(b, s, n, l, f, d, st, is, li); rows[i] = t; } - - StructObjectInspector rowOI = (StructObjectInspector)ObjectInspectorFactory - .getReflectionObjectInspector(MyTestClass.class, - ObjectInspectorOptions.JAVA); - + + StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory + .getReflectionObjectInspector(MyTestClass.class, + ObjectInspectorOptions.JAVA); + String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI); String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI); - + // call the tests // 1/ test LazyBinarySerDe testLazyBinarySerDe(rows, rowOI, getSerDe(fieldNames, fieldTypes)); @@ -453,12 +503,12 @@ // 3/ test serialization and deserialization with different schemas testShorterSchemaDeserialization(r); // 4/ test serialization and deserialization with different schemas - testLongerSchemaDeserialization(r); + testLongerSchemaDeserialization(r); // 5/ test serialization and deserialization with different schemas testShorterSchemaDeserialization1(r); // 6/ test serialization and deserialization with different schemas - testLongerSchemaDeserialization1(r); - + testLongerSchemaDeserialization1(r); + System.out.println("Test TestLazyBinarySerDe passed!"); } catch (Throwable e) { e.printStackTrace(); Index: serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java (working copy) @@ -33,22 +33,22 @@ MyTestInnerStruct myStruct; List myList; Map> myMap; - - public MyTestClassBigger() { + + public MyTestClassBigger() { } - - public MyTestClassBigger(Byte b, Short s, Integer i, Long l, - Float f, Double d, String st, MyTestInnerStruct is, - List li, Map> mp) { - this.myByte = b; - this.myShort = s; - this.myInt = i; - this.myLong = l; - this.myFloat = f; - this.myDouble = d; - this.myString = st; - this.myStruct = is; - this.myList = li; - this.myMap = mp; + + public MyTestClassBigger(Byte b, Short s, Integer i, Long l, Float f, + Double d, String st, MyTestInnerStruct is, List li, + Map> mp) { + myByte = b; + myShort = s; + myInt = i; + myLong = l; + myFloat = f; + myDouble = d; + myString = st; + myStruct = is; + myList = li; + myMap = mp; } } Index: serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java (working copy) @@ -28,19 +28,19 @@ Double myDouble; String myString; MyTestInnerStruct myStruct; - - public MyTestClassSmaller() { + + public MyTestClassSmaller() { } - - public MyTestClassSmaller(Byte b, Short s, Integer i, Long l, - Float f, Double d, String st, MyTestInnerStruct is) { - this.myByte = b; - this.myShort = s; - this.myInt = i; - this.myLong = l; - this.myFloat = f; - this.myDouble = d; - this.myString = st; - this.myStruct = is; + + public MyTestClassSmaller(Byte b, Short s, Integer i, Long l, Float f, + Double d, String st, MyTestInnerStruct is) { + myByte = b; + myShort = s; + myInt = i; + myLong = l; + myFloat = f; + myDouble = d; + myString = st; + myStruct = is; } } Index: serde/src/test/org/apache/hadoop/hive/serde2/dynamic_type/TestDynamicSerDe.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/dynamic_type/TestDynamicSerDe.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/dynamic_type/TestDynamicSerDe.java (working copy) @@ -18,33 +18,31 @@ package org.apache.hadoop.hive.serde2.dynamic_type; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.List; import java.util.Properties; import java.util.Random; import java.util.Map.Entry; +import junit.framework.TestCase; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde.Constants; -import org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol; -import org.apache.hadoop.hive.serde.Constants; - -import junit.framework.TestCase; import org.apache.hadoop.io.BytesWritable; public class TestDynamicSerDe extends TestCase { - public static HashMap makeHashMap(String... params) { - HashMap r = new HashMap(); - for(int i=0; i makeHashMap(String... params) { + HashMap r = new HashMap(); + for (int i = 0; i < params.length; i += 2) { + r.put(params[i], params[i + 1]); } return r; } + public void testDynamicSerDe() throws Throwable { try { @@ -52,7 +50,7 @@ ArrayList bye = new ArrayList(); bye.add("firstString"); bye.add("secondString"); - HashMap another = new HashMap(); + HashMap another = new HashMap(); another.put("firstKey", 1); another.put("secondKey", 2); ArrayList struct = new ArrayList(); @@ -62,25 +60,27 @@ struct.add(Integer.valueOf(-234)); struct.add(Double.valueOf(1.0)); struct.add(Double.valueOf(-2.5)); - - + // All protocols ArrayList protocols = new ArrayList(); ArrayList isBinaries = new ArrayList(); - ArrayList> additionalParams = new ArrayList>(); + ArrayList> additionalParams = new ArrayList>(); - protocols.add(org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol.class.getName()); + protocols + .add(org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol.class + .getName()); isBinaries.add(true); additionalParams.add(makeHashMap("serialization.sort.order", "++++++")); - protocols.add(org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol.class.getName()); + protocols + .add(org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol.class + .getName()); isBinaries.add(true); additionalParams.add(makeHashMap("serialization.sort.order", "------")); - protocols.add(org.apache.thrift.protocol.TBinaryProtocol.class.getName()); isBinaries.add(true); additionalParams.add(null); - + protocols.add(org.apache.thrift.protocol.TJSONProtocol.class.getName()); isBinaries.add(false); additionalParams.add(null); @@ -89,91 +89,103 @@ // protocols.add(org.apache.thrift.protocol.TSimpleJSONProtocol.class.getName()); // isBinaries.add(false); // additionalParams.add(null); - + // TCTLSeparatedProtocol is not done yet. - protocols.add(org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName()); + protocols + .add(org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class + .getName()); isBinaries.add(false); additionalParams.add(null); - + System.out.println("input struct = " + struct); - - for(int pp = 0; pp 2bye, map another, i32 nhello, double d, double nd}"); - schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString()); + schema.setProperty( + org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, + "test"); + schema + .setProperty( + Constants.SERIALIZATION_DDL, + "struct test { i32 _hello, list 2bye, map another, i32 nhello, double d, double nd}"); + schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe() + .getClass().toString()); HashMap p = additionalParams.get(pp); if (p != null) { - for(Entry e: p.entrySet()) { + for (Entry e : p.entrySet()) { schema.setProperty(e.getKey(), e.getValue()); } } - + DynamicSerDe serde = new DynamicSerDe(); serde.initialize(new Configuration(), schema); - + // Try getObjectInspector ObjectInspector oi = serde.getObjectInspector(); System.out.println("TypeName = " + oi.getTypeName()); - // Try to serialize BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi); - System.out.println("bytes =" + hexString(bytes)); - + System.out.println("bytes =" + hexString(bytes)); + if (!isBinary) { - System.out.println("bytes in text =" + new String(bytes.get(), 0, bytes.getSize())); + System.out.println("bytes in text =" + + new String(bytes.get(), 0, bytes.getSize())); } - + // Try to deserialize Object o = serde.deserialize(bytes); System.out.println("o class = " + o.getClass()); - List olist = (List)o; + List olist = (List) o; System.out.println("o size = " + olist.size()); System.out.println("o[0] class = " + olist.get(0).getClass()); System.out.println("o[1] class = " + olist.get(1).getClass()); System.out.println("o[2] class = " + olist.get(2).getClass()); System.out.println("o = " + o); - + assertEquals(struct, o); } - + } catch (Throwable e) { e.printStackTrace(); throw e; } - + } + public String hexString(BytesWritable bytes) { StringBuilder sb = new StringBuilder(); - for (int i=0; i0) { - int compareResult = bytes[i-1].compareTo(bytes[i]); - if ( (compareResult<0 && !ascending) || (compareResult>0 && ascending) ) { - System.out.println("Test failed in " + (ascending ? "ascending" : "descending") + " order."); - System.out.println("serialized data of " + structs[i-1] + " = " + hexString(bytes[i-1])); - System.out.println("serialized data of " + structs[i] + " = " + hexString(bytes[i])); - fail("Sort order of serialized " + structs[i-1] + " and " + structs[i] + " are reversed!"); + if (i > 0) { + int compareResult = bytes[i - 1].compareTo(bytes[i]); + if ((compareResult < 0 && !ascending) + || (compareResult > 0 && ascending)) { + System.out.println("Test failed in " + + (ascending ? "ascending" : "descending") + " order."); + System.out.println("serialized data of " + structs[i - 1] + " = " + + hexString(bytes[i - 1])); + System.out.println("serialized data of " + structs[i] + " = " + + hexString(bytes[i])); + fail("Sort order of serialized " + structs[i - 1] + " and " + + structs[i] + " are reversed!"); } } } // Try to deserialize Object[] deserialized = new Object[structs.length]; - for (int i=0; i struct = new ArrayList(); - if (i==0) { + if (i == 0) { struct.add(null); } else { - struct.add(Double.valueOf((r.nextDouble()-0.5)*10)); + struct.add(Double.valueOf((r.nextDouble() - 0.5) * 10)); } structs[i] = struct; } @@ -274,86 +306,90 @@ testTBinarySortableProtocol(structs, ddl, false); // Test integer - for (int i=0; i struct = new ArrayList(); - if (i==0) { + if (i == 0) { struct.add(null); } else { - struct.add((int)((r.nextDouble()-0.5)*1.5*Integer.MAX_VALUE)); + struct.add((int) ((r.nextDouble() - 0.5) * 1.5 * Integer.MAX_VALUE)); } structs[i] = struct; } sort(structs); - // Null should be smaller than any other value, so put a null at the front end + // Null should be smaller than any other value, so put a null at the front + // end // to test whether that is held. - ((List)structs[0]).set(0, null); + ((List) structs[0]).set(0, null); ddl = "struct test { i32 hello}"; System.out.println("Testing " + ddl); testTBinarySortableProtocol(structs, ddl, true); testTBinarySortableProtocol(structs, ddl, false); // Test long - for (int i=0; i struct = new ArrayList(); - if (i==0) { + if (i == 0) { struct.add(null); } else { - struct.add((long)((r.nextDouble()-0.5)*1.5*Long.MAX_VALUE)); + struct.add((long) ((r.nextDouble() - 0.5) * 1.5 * Long.MAX_VALUE)); } structs[i] = struct; } sort(structs); - // Null should be smaller than any other value, so put a null at the front end + // Null should be smaller than any other value, so put a null at the front + // end // to test whether that is held. - ((List)structs[0]).set(0, null); + ((List) structs[0]).set(0, null); ddl = "struct test { i64 hello}"; System.out.println("Testing " + ddl); testTBinarySortableProtocol(structs, ddl, true); testTBinarySortableProtocol(structs, ddl, false); // Test string - for (int i=0; i struct = new ArrayList(); - if (i==0) { + if (i == 0) { struct.add(null); } else { - struct.add(String.valueOf((r.nextDouble()-0.5)*1000)); + struct.add(String.valueOf((r.nextDouble() - 0.5) * 1000)); } structs[i] = struct; } sort(structs); - // Null should be smaller than any other value, so put a null at the front end + // Null should be smaller than any other value, so put a null at the front + // end // to test whether that is held. - ((List)structs[0]).set(0, null); + ((List) structs[0]).set(0, null); ddl = "struct test { string hello}"; System.out.println("Testing " + ddl); testTBinarySortableProtocol(structs, ddl, true); testTBinarySortableProtocol(structs, ddl, false); // Test string + double - for (int i=0; i struct = new ArrayList(); - if (i%9==0) { + if (i % 9 == 0) { struct.add(null); } else { - struct.add("str" + (i/5)); + struct.add("str" + (i / 5)); } - if (i%7==0) { + if (i % 7 == 0) { struct.add(null); } else { - struct.add(Double.valueOf((r.nextDouble()-0.5)*10)); + struct.add(Double.valueOf((r.nextDouble() - 0.5) * 10)); } structs[i] = struct; } sort(structs); - // Null should be smaller than any other value, so put a null at the front end + // Null should be smaller than any other value, so put a null at the front + // end // to test whether that is held. - ((List)structs[0]).set(0, null); + ((List) structs[0]).set(0, null); ddl = "struct test { string hello, double another}"; System.out.println("Testing " + ddl); testTBinarySortableProtocol(structs, ddl, true); testTBinarySortableProtocol(structs, ddl, false); - + System.out.println("Test testTBinarySortableProtocol passed!"); } catch (Throwable e) { e.printStackTrace(); @@ -361,16 +397,14 @@ } } - public void testConfigurableTCTLSeparated() throws Throwable { try { - // Try to construct an object ArrayList bye = new ArrayList(); bye.add("firstString"); bye.add("secondString"); - HashMap another = new HashMap(); + HashMap another = new HashMap(); another.put("firstKey", 1); another.put("secondKey", 2); ArrayList struct = new ArrayList(); @@ -379,11 +413,17 @@ struct.add(another); Properties schema = new Properties(); - schema.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName()); - schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test"); - schema.setProperty(Constants.SERIALIZATION_DDL, - "struct test { i32 hello, list bye, map another}"); - schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString()); + schema.setProperty(Constants.SERIALIZATION_FORMAT, + org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class + .getName()); + schema.setProperty( + org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, + "test"); + schema + .setProperty(Constants.SERIALIZATION_DDL, + "struct test { i32 hello, list bye, map another}"); + schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe() + .getClass().toString()); schema.setProperty(Constants.FIELD_DELIM, "9"); schema.setProperty(Constants.COLLECTION_DELIM, "1"); @@ -393,42 +433,44 @@ DynamicSerDe serde = new DynamicSerDe(); serde.initialize(new Configuration(), schema); - TCTLSeparatedProtocol prot = (TCTLSeparatedProtocol)serde.oprot_; + TCTLSeparatedProtocol prot = (TCTLSeparatedProtocol) serde.oprot_; assertTrue(prot.getPrimarySeparator().equals("\u0009")); - + ObjectInspector oi = serde.getObjectInspector(); // Try to serialize BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi); - + hexString(bytes); - String compare = "234" + "\u0009" + "firstString" + "\u0001" + "secondString" + "\u0009" + "firstKey" + "\u0004" + "1" + "\u0001" + "secondKey" + "\u0004" + "2"; + String compare = "234" + "\u0009" + "firstString" + "\u0001" + + "secondString" + "\u0009" + "firstKey" + "\u0004" + "1" + "\u0001" + + "secondKey" + "\u0004" + "2"; - System.out.println("bytes in text =" + new String(bytes.get(), 0, bytes.getSize()) + ">"); + System.out.println("bytes in text =" + + new String(bytes.get(), 0, bytes.getSize()) + ">"); System.out.println("compare to =" + compare + ">"); - - assertTrue(compare.equals( new String(bytes.get(), 0, bytes.getSize()))); - + + assertTrue(compare.equals(new String(bytes.get(), 0, bytes.getSize()))); + // Try to deserialize Object o = serde.deserialize(bytes); System.out.println("o class = " + o.getClass()); - List olist = (List)o; + List olist = (List) o; System.out.println("o size = " + olist.size()); System.out.println("o[0] class = " + olist.get(0).getClass()); System.out.println("o[1] class = " + olist.get(1).getClass()); System.out.println("o[2] class = " + olist.get(2).getClass()); System.out.println("o = " + o); - + assertEquals(o, struct); - + } catch (Throwable e) { e.printStackTrace(); throw e; } - - } + } /** * Tests a single null list within a struct with return nulls on @@ -437,10 +479,9 @@ public void testNulls1() throws Throwable { try { - // Try to construct an object ArrayList bye = null; - HashMap another = new HashMap(); + HashMap another = new HashMap(); another.put("firstKey", 1); another.put("secondKey", 2); ArrayList struct = new ArrayList(); @@ -449,34 +490,40 @@ struct.add(another); Properties schema = new Properties(); - schema.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName()); - schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test"); - schema.setProperty(Constants.SERIALIZATION_DDL, - "struct test { i32 hello, list bye, map another}"); - schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString()); + schema.setProperty(Constants.SERIALIZATION_FORMAT, + org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class + .getName()); + schema.setProperty( + org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, + "test"); + schema + .setProperty(Constants.SERIALIZATION_DDL, + "struct test { i32 hello, list bye, map another}"); + schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe() + .getClass().toString()); schema.setProperty(TCTLSeparatedProtocol.ReturnNullsKey, "true"); DynamicSerDe serde = new DynamicSerDe(); serde.initialize(new Configuration(), schema); ObjectInspector oi = serde.getObjectInspector(); - + // Try to serialize BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi); - + hexString(bytes); // Try to deserialize Object o = serde.deserialize(bytes); assertEquals(struct, o); - + } catch (Throwable e) { e.printStackTrace(); throw e; } - - } + } + /** * Tests all elements of a struct being null with return nulls on */ @@ -484,7 +531,6 @@ public void testNulls2() throws Throwable { try { - // Try to construct an object ArrayList bye = null; HashMap another = null; @@ -494,11 +540,17 @@ struct.add(another); Properties schema = new Properties(); - schema.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName()); - schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test"); - schema.setProperty(Constants.SERIALIZATION_DDL, - "struct test { i32 hello, list bye, map another}"); - schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString()); + schema.setProperty(Constants.SERIALIZATION_FORMAT, + org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class + .getName()); + schema.setProperty( + org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, + "test"); + schema + .setProperty(Constants.SERIALIZATION_DDL, + "struct test { i32 hello, list bye, map another}"); + schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe() + .getClass().toString()); schema.setProperty(TCTLSeparatedProtocol.ReturnNullsKey, "true"); DynamicSerDe serde = new DynamicSerDe(); @@ -508,27 +560,28 @@ // Try to serialize BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi); - + hexString(bytes); // Try to deserialize Object o = serde.deserialize(bytes); - List olist = (List)o; + List olist = (List) o; assertTrue(olist.size() == 3); assertEquals(null, olist.get(0)); assertEquals(null, olist.get(1)); assertEquals(null, olist.get(2)); - - // assertEquals(o, struct); Cannot do this because types of null lists are wrong. - + + // assertEquals(o, struct); Cannot do this because types of null lists are + // wrong. + } catch (Throwable e) { e.printStackTrace(); throw e; } - - } + } + /** * Tests map and list being empty with return nulls on */ @@ -536,9 +589,8 @@ public void testNulls3() throws Throwable { try { - // Try to construct an object - ArrayList bye = new ArrayList (); + ArrayList bye = new ArrayList(); HashMap another = null; ArrayList struct = new ArrayList(); struct.add(null); @@ -546,11 +598,17 @@ struct.add(another); Properties schema = new Properties(); - schema.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName()); - schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test"); - schema.setProperty(Constants.SERIALIZATION_DDL, - "struct test { i32 hello, list bye, map another}"); - schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString()); + schema.setProperty(Constants.SERIALIZATION_FORMAT, + org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class + .getName()); + schema.setProperty( + org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, + "test"); + schema + .setProperty(Constants.SERIALIZATION_DDL, + "struct test { i32 hello, list bye, map another}"); + schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe() + .getClass().toString()); schema.setProperty(TCTLSeparatedProtocol.ReturnNullsKey, "true"); DynamicSerDe serde = new DynamicSerDe(); @@ -560,27 +618,27 @@ // Try to serialize BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi); - + hexString(bytes); // Try to deserialize Object o = serde.deserialize(bytes); - List olist = (List)o; + List olist = (List) o; assertTrue(olist.size() == 3); assertEquals(null, olist.get(0)); - assertEquals(0, ((List)olist.get(1)).size()); + assertEquals(0, ((List) olist.get(1)).size()); assertEquals(null, olist.get(2)); - - // assertEquals(o, struct); Cannot do this because types of null lists are wrong. - + + // assertEquals(o, struct); Cannot do this because types of null lists are + // wrong. + } catch (Throwable e) { e.printStackTrace(); throw e; } - - } + } /** * Tests map and list null/empty with return nulls *off* @@ -589,9 +647,8 @@ public void testNulls4() throws Throwable { try { - // Try to construct an object - ArrayList bye = new ArrayList (); + ArrayList bye = new ArrayList(); HashMap another = null; ArrayList struct = new ArrayList(); struct.add(null); @@ -599,11 +656,17 @@ struct.add(another); Properties schema = new Properties(); - schema.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName()); - schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test"); - schema.setProperty(Constants.SERIALIZATION_DDL, - "struct test { i32 hello, list bye, map another}"); - schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString()); + schema.setProperty(Constants.SERIALIZATION_FORMAT, + org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class + .getName()); + schema.setProperty( + org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, + "test"); + schema + .setProperty(Constants.SERIALIZATION_DDL, + "struct test { i32 hello, list bye, map another}"); + schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe() + .getClass().toString()); schema.setProperty(TCTLSeparatedProtocol.ReturnNullsKey, "false"); DynamicSerDe serde = new DynamicSerDe(); @@ -613,29 +676,29 @@ // Try to serialize BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi); - + hexString(bytes); // Try to deserialize Object o = serde.deserialize(bytes); - List olist = (List)o; + List olist = (List) o; assertTrue(olist.size() == 3); - assertEquals(new Integer(0), (Integer)olist.get(0)); - List num1 = (List)olist.get(1); + assertEquals(new Integer(0), (Integer) olist.get(0)); + List num1 = (List) olist.get(1); assertTrue(num1.size() == 0); - Map num2 = (Map)olist.get(2); + Map num2 = (Map) olist.get(2); assertTrue(num2.size() == 0); - - // assertEquals(o, struct); Cannot do this because types of null lists are wrong. - + + // assertEquals(o, struct); Cannot do this because types of null lists are + // wrong. + } catch (Throwable e) { e.printStackTrace(); throw e; } - - } + } /** * Tests map and list null/empty with return nulls *off* @@ -644,16 +707,21 @@ public void testStructsinStructs() throws Throwable { try { - Properties schema = new Properties(); - // schema.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.thrift.protocol.TJSONProtocol.class.getName()); - schema.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.thrift.protocol.TBinaryProtocol.class.getName()); - schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test"); - schema.setProperty(Constants.SERIALIZATION_DDL, - "struct inner { i32 field1, string field2 },struct test {inner foo, i32 hello, list bye, map another}"); - schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString()); + // schema.setProperty(Constants.SERIALIZATION_FORMAT, + // org.apache.thrift.protocol.TJSONProtocol.class.getName()); + schema.setProperty(Constants.SERIALIZATION_FORMAT, + org.apache.thrift.protocol.TBinaryProtocol.class.getName()); + schema.setProperty( + org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, + "test"); + schema + .setProperty( + Constants.SERIALIZATION_DDL, + "struct inner { i32 field1, string field2 },struct test {inner foo, i32 hello, list bye, map another}"); + schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe() + .getClass().toString()); - // // construct object of above type // @@ -664,10 +732,10 @@ innerStruct.add(new String("hello world")); // construct outer struct - ArrayList bye = new ArrayList (); + ArrayList bye = new ArrayList(); bye.add("firstString"); bye.add("secondString"); - HashMap another = new HashMap(); + HashMap another = new HashMap(); another.put("firstKey", 1); another.put("secondKey", 2); @@ -688,9 +756,8 @@ // Try to deserialize Object o = serde.deserialize(bytes); - List olist = (List)o; + List olist = (List) o; - assertEquals(4, olist.size()); assertEquals(innerStruct, olist.get(0)); assertEquals(new Integer(234), olist.get(1)); @@ -701,12 +768,9 @@ e.printStackTrace(); throw e; } - - } + } - - public void testSkip() throws Throwable { try { @@ -714,7 +778,7 @@ ArrayList bye = new ArrayList(); bye.add("firstString"); bye.add("secondString"); - HashMap another = new HashMap(); + HashMap another = new HashMap(); another.put("firstKey", 1); another.put("secondKey", 2); ArrayList struct = new ArrayList(); @@ -723,11 +787,17 @@ struct.add(another); Properties schema = new Properties(); - schema.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName()); - schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test"); - schema.setProperty(Constants.SERIALIZATION_DDL, - "struct test { i32 hello, list bye, map another}"); - schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString()); + schema.setProperty(Constants.SERIALIZATION_FORMAT, + org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class + .getName()); + schema.setProperty( + org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, + "test"); + schema + .setProperty(Constants.SERIALIZATION_DDL, + "struct test { i32 hello, list bye, map another}"); + schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe() + .getClass().toString()); schema.setProperty(Constants.FIELD_DELIM, "9"); schema.setProperty(Constants.COLLECTION_DELIM, "1"); @@ -737,47 +807,51 @@ DynamicSerDe serde = new DynamicSerDe(); serde.initialize(new Configuration(), schema); - TCTLSeparatedProtocol prot = (TCTLSeparatedProtocol)serde.oprot_; + TCTLSeparatedProtocol prot = (TCTLSeparatedProtocol) serde.oprot_; assertTrue(prot.getPrimarySeparator().equals("\u0009")); - + ObjectInspector oi = serde.getObjectInspector(); // Try to serialize BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi); - + hexString(bytes); - String compare = "234" + "\u0009" + "firstString" + "\u0001" + "secondString" + "\u0009" + "firstKey" + "\u0004" + "1" + "\u0001" + "secondKey" + "\u0004" + "2"; + String compare = "234" + "\u0009" + "firstString" + "\u0001" + + "secondString" + "\u0009" + "firstKey" + "\u0004" + "1" + "\u0001" + + "secondKey" + "\u0004" + "2"; - System.out.println("bytes in text =" + new String(bytes.get(), 0, bytes.getSize()) + ">"); + System.out.println("bytes in text =" + + new String(bytes.get(), 0, bytes.getSize()) + ">"); System.out.println("compare to =" + compare + ">"); - - assertTrue(compare.equals( new String(bytes.get(), 0, bytes.getSize()))); - - schema.setProperty(Constants.SERIALIZATION_DDL, - "struct test { i32 hello, skip list bye, map another}"); + assertTrue(compare.equals(new String(bytes.get(), 0, bytes.getSize()))); + + schema + .setProperty(Constants.SERIALIZATION_DDL, + "struct test { i32 hello, skip list bye, map another}"); + serde.initialize(new Configuration(), schema); // Try to deserialize Object o = serde.deserialize(bytes); System.out.println("o class = " + o.getClass()); - List olist = (List)o; + List olist = (List) o; System.out.println("o size = " + olist.size()); System.out.println("o = " + o); - + assertEquals(null, olist.get(1)); // set the skipped field to null - struct.set(1,null); + struct.set(1, null); assertEquals(o, struct); - + } catch (Throwable e) { e.printStackTrace(); throw e; } - - } + } + } Index: serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java (working copy) @@ -21,6 +21,8 @@ import java.util.HashMap; import java.util.List; +import junit.framework.TestCase; + import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -34,38 +36,41 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; -import junit.framework.TestCase; - public class TestStandardObjectInspectors extends TestCase { - - void doTestStandardPrimitiveObjectInspector(Class writableClass, Class javaClass) throws Throwable { + void doTestStandardPrimitiveObjectInspector(Class writableClass, + Class javaClass) throws Throwable { try { - PrimitiveObjectInspector oi1 = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( - PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveWritableClass(writableClass).primitiveCategory); - PrimitiveObjectInspector oi2 = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( - PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveWritableClass(writableClass).primitiveCategory); + PrimitiveObjectInspector oi1 = PrimitiveObjectInspectorFactory + .getPrimitiveWritableObjectInspector(PrimitiveObjectInspectorUtils + .getTypeEntryFromPrimitiveWritableClass(writableClass).primitiveCategory); + PrimitiveObjectInspector oi2 = PrimitiveObjectInspectorFactory + .getPrimitiveWritableObjectInspector(PrimitiveObjectInspectorUtils + .getTypeEntryFromPrimitiveWritableClass(writableClass).primitiveCategory); assertEquals(oi1, oi2); assertEquals(Category.PRIMITIVE, oi1.getCategory()); assertEquals(writableClass, oi1.getPrimitiveWritableClass()); assertEquals(javaClass, oi1.getJavaPrimitiveClass()); // Cannot create NullWritable instances if (!NullWritable.class.equals(writableClass)) { - assertEquals(writableClass, oi1.getPrimitiveWritableObject(writableClass.newInstance()).getClass()); - assertEquals(javaClass, oi1.getPrimitiveJavaObject(writableClass.newInstance()).getClass()); + assertEquals(writableClass, oi1.getPrimitiveWritableObject( + writableClass.newInstance()).getClass()); + assertEquals(javaClass, oi1.getPrimitiveJavaObject( + writableClass.newInstance()).getClass()); } - assertEquals(PrimitiveObjectInspectorUtils.getTypeNameFromPrimitiveWritable(writableClass), - oi1.getTypeName()); + assertEquals(PrimitiveObjectInspectorUtils + .getTypeNameFromPrimitiveWritable(writableClass), oi1.getTypeName()); } catch (Throwable e) { e.printStackTrace(); throw e; } } - + public void testStandardPrimitiveObjectInspector() throws Throwable { try { doTestStandardPrimitiveObjectInspector(NullWritable.class, Void.class); - doTestStandardPrimitiveObjectInspector(BooleanWritable.class, Boolean.class); + doTestStandardPrimitiveObjectInspector(BooleanWritable.class, + Boolean.class); doTestStandardPrimitiveObjectInspector(ByteWritable.class, Byte.class); doTestStandardPrimitiveObjectInspector(ShortWritable.class, Short.class); doTestStandardPrimitiveObjectInspector(IntWritable.class, Integer.class); @@ -79,41 +84,52 @@ } } - void doTestJavaPrimitiveObjectInspector(Class writableClass, Class javaClass, Object javaObject) throws Throwable { + void doTestJavaPrimitiveObjectInspector(Class writableClass, + Class javaClass, Object javaObject) throws Throwable { try { - PrimitiveObjectInspector oi1 = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( - PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveJavaClass(javaClass).primitiveCategory); - PrimitiveObjectInspector oi2 = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( - PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveJavaClass(javaClass).primitiveCategory); + PrimitiveObjectInspector oi1 = PrimitiveObjectInspectorFactory + .getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils + .getTypeEntryFromPrimitiveJavaClass(javaClass).primitiveCategory); + PrimitiveObjectInspector oi2 = PrimitiveObjectInspectorFactory + .getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils + .getTypeEntryFromPrimitiveJavaClass(javaClass).primitiveCategory); assertEquals(oi1, oi2); assertEquals(Category.PRIMITIVE, oi1.getCategory()); assertEquals(javaClass, oi1.getJavaPrimitiveClass()); assertEquals(writableClass, oi1.getPrimitiveWritableClass()); if (javaObject != null) { - assertEquals(javaClass, oi1.getPrimitiveJavaObject(javaObject).getClass()); - assertEquals(writableClass, oi1.getPrimitiveWritableObject(javaObject).getClass()); + assertEquals(javaClass, oi1.getPrimitiveJavaObject(javaObject) + .getClass()); + assertEquals(writableClass, oi1.getPrimitiveWritableObject(javaObject) + .getClass()); } - - assertEquals(PrimitiveObjectInspectorUtils.getTypeNameFromPrimitiveJava(javaClass), - oi1.getTypeName()); + + assertEquals(PrimitiveObjectInspectorUtils + .getTypeNameFromPrimitiveJava(javaClass), oi1.getTypeName()); } catch (Throwable e) { e.printStackTrace(); throw e; } } - + public void testJavaPrimitiveObjectInspector() throws Throwable { try { doTestJavaPrimitiveObjectInspector(NullWritable.class, Void.class, null); - doTestJavaPrimitiveObjectInspector(BooleanWritable.class, Boolean.class, true); - doTestJavaPrimitiveObjectInspector(ByteWritable.class, Byte.class, (byte)1); - doTestJavaPrimitiveObjectInspector(ShortWritable.class, Short.class, (short)1); - doTestJavaPrimitiveObjectInspector(IntWritable.class, Integer.class, (int)1); - doTestJavaPrimitiveObjectInspector(LongWritable.class, Long.class, (long)1); - doTestJavaPrimitiveObjectInspector(FloatWritable.class, Float.class, (float)1); - doTestJavaPrimitiveObjectInspector(DoubleWritable.class, Double.class, (double)1); + doTestJavaPrimitiveObjectInspector(BooleanWritable.class, Boolean.class, + true); + doTestJavaPrimitiveObjectInspector(ByteWritable.class, Byte.class, + (byte) 1); + doTestJavaPrimitiveObjectInspector(ShortWritable.class, Short.class, + (short) 1); + doTestJavaPrimitiveObjectInspector(IntWritable.class, Integer.class, 1); + doTestJavaPrimitiveObjectInspector(LongWritable.class, Long.class, + (long) 1); + doTestJavaPrimitiveObjectInspector(FloatWritable.class, Float.class, + (float) 1); + doTestJavaPrimitiveObjectInspector(DoubleWritable.class, Double.class, + (double) 1); doTestJavaPrimitiveObjectInspector(Text.class, String.class, "a"); - + } catch (Throwable e) { e.printStackTrace(); throw e; @@ -122,23 +138,26 @@ public void testStandardListObjectInspector() throws Throwable { try { - StandardListObjectInspector loi1 = ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.javaIntObjectInspector); - StandardListObjectInspector loi2 = ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.javaIntObjectInspector); + StandardListObjectInspector loi1 = ObjectInspectorFactory + .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector); + StandardListObjectInspector loi2 = ObjectInspectorFactory + .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector); assertEquals(loi1, loi2); - + // metadata assertEquals(Category.LIST, loi1.getCategory()); - assertEquals(PrimitiveObjectInspectorFactory.javaIntObjectInspector, - loi1.getListElementObjectInspector()); - + assertEquals(PrimitiveObjectInspectorFactory.javaIntObjectInspector, loi1 + .getListElementObjectInspector()); + // null assertNull("loi1.getList(null) should be null.", loi1.getList(null)); - assertEquals("loi1.getListLength(null) should be -1.", loi1.getListLength(null), -1); - assertNull("loi1.getListElement(null, 0) should be null", loi1.getListElement(null, 0)); - assertNull("loi1.getListElement(null, 100) should be null", loi1.getListElement(null, 100)); - + assertEquals("loi1.getListLength(null) should be -1.", loi1 + .getListLength(null), -1); + assertNull("loi1.getListElement(null, 0) should be null", loi1 + .getListElement(null, 0)); + assertNull("loi1.getListElement(null, 100) should be null", loi1 + .getListElement(null, 100)); + // ArrayList ArrayList list = new ArrayList(); list.add(0); @@ -151,7 +170,7 @@ assertEquals(3, loi1.getListElement(list, 3)); assertNull(loi1.getListElement(list, -1)); assertNull(loi1.getListElement(list, 4)); - + // Settable Object list4 = loi1.create(4); loi1.set(list4, 0, 0); @@ -159,48 +178,50 @@ loi1.set(list4, 2, 2); loi1.set(list4, 3, 3); assertEquals(list, list4); - + loi1.resize(list4, 5); loi1.set(list4, 4, 4); loi1.resize(list4, 4); assertEquals(list, list4); - + } catch (Throwable e) { e.printStackTrace(); throw e; } - + } public void testStandardMapObjectInspector() throws Throwable { try { - StandardMapObjectInspector moi1 = ObjectInspectorFactory.getStandardMapObjectInspector( - PrimitiveObjectInspectorFactory.javaStringObjectInspector, - PrimitiveObjectInspectorFactory.javaIntObjectInspector - ); - StandardMapObjectInspector moi2 = ObjectInspectorFactory.getStandardMapObjectInspector( - PrimitiveObjectInspectorFactory.javaStringObjectInspector, - PrimitiveObjectInspectorFactory.javaIntObjectInspector - ); + StandardMapObjectInspector moi1 = ObjectInspectorFactory + .getStandardMapObjectInspector( + PrimitiveObjectInspectorFactory.javaStringObjectInspector, + PrimitiveObjectInspectorFactory.javaIntObjectInspector); + StandardMapObjectInspector moi2 = ObjectInspectorFactory + .getStandardMapObjectInspector( + PrimitiveObjectInspectorFactory.javaStringObjectInspector, + PrimitiveObjectInspectorFactory.javaIntObjectInspector); assertEquals(moi1, moi2); - + // metadata assertEquals(Category.MAP, moi1.getCategory()); - assertEquals(moi1.getMapKeyObjectInspector(), + assertEquals(moi1.getMapKeyObjectInspector(), PrimitiveObjectInspectorFactory.javaStringObjectInspector); - assertEquals(moi2.getMapValueObjectInspector(), + assertEquals(moi2.getMapValueObjectInspector(), PrimitiveObjectInspectorFactory.javaIntObjectInspector); - + // null assertNull(moi1.getMap(null)); assertNull(moi1.getMapValueElement(null, null)); assertNull(moi1.getMapValueElement(null, "nokey")); assertEquals(-1, moi1.getMapSize(null)); - assertEquals("map<" - + PrimitiveObjectInspectorFactory.javaStringObjectInspector.getTypeName() + "," - + PrimitiveObjectInspectorFactory.javaIntObjectInspector.getTypeName() + ">", - moi1.getTypeName()); - + assertEquals("map<" + + PrimitiveObjectInspectorFactory.javaStringObjectInspector + .getTypeName() + + "," + + PrimitiveObjectInspectorFactory.javaIntObjectInspector + .getTypeName() + ">", moi1.getTypeName()); + // HashMap HashMap map = new HashMap(); map.put("one", 1); @@ -213,7 +234,7 @@ assertEquals(3, moi1.getMapValueElement(map, "three")); assertNull(moi1.getMapValueElement(map, null)); assertNull(moi1.getMapValueElement(map, "null")); - + // Settable Object map3 = moi1.create(); moi1.put(map3, "one", 1); @@ -222,15 +243,14 @@ assertEquals(map, map3); moi1.clear(map3); assertEquals(0, moi1.getMapSize(map3)); - + } catch (Throwable e) { e.printStackTrace(); throw e; } - + } - @SuppressWarnings("unchecked") public void testStandardStructObjectInspector() throws Throwable { try { @@ -239,52 +259,62 @@ fieldNames.add("secondString"); fieldNames.add("thirdBoolean"); ArrayList fieldObjectInspectors = new ArrayList(); - fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector); - fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); - fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector); - - StandardStructObjectInspector soi1 = ObjectInspectorFactory.getStandardStructObjectInspector( - fieldNames, fieldObjectInspectors); - StandardStructObjectInspector soi2 = ObjectInspectorFactory.getStandardStructObjectInspector( - (ArrayList)fieldNames.clone(), (ArrayList)fieldObjectInspectors.clone()); + fieldObjectInspectors + .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector); + fieldObjectInspectors + .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); + fieldObjectInspectors + .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector); + + StandardStructObjectInspector soi1 = ObjectInspectorFactory + .getStandardStructObjectInspector(fieldNames, fieldObjectInspectors); + StandardStructObjectInspector soi2 = ObjectInspectorFactory + .getStandardStructObjectInspector((ArrayList) fieldNames + .clone(), (ArrayList) fieldObjectInspectors + .clone()); assertEquals(soi1, soi2); - + // metadata assertEquals(Category.STRUCT, soi1.getCategory()); List fields = soi1.getAllStructFieldRefs(); assertEquals(3, fields.size()); - for (int i=0; i<3; i++) { - assertEquals(fieldNames.get(i).toLowerCase(), fields.get(i).getFieldName()); - assertEquals(fieldObjectInspectors.get(i), fields.get(i).getFieldObjectInspector()); + for (int i = 0; i < 3; i++) { + assertEquals(fieldNames.get(i).toLowerCase(), fields.get(i) + .getFieldName()); + assertEquals(fieldObjectInspectors.get(i), fields.get(i) + .getFieldObjectInspector()); } assertEquals(fields.get(1), soi1.getStructFieldRef("secondString")); - StringBuilder structTypeName = new StringBuilder(); + StringBuilder structTypeName = new StringBuilder(); structTypeName.append("struct<"); - for(int i=0; i0) structTypeName.append(","); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + structTypeName.append(","); + } structTypeName.append(fields.get(i).getFieldName()); structTypeName.append(":"); - structTypeName.append(fields.get(i).getFieldObjectInspector().getTypeName()); + structTypeName.append(fields.get(i).getFieldObjectInspector() + .getTypeName()); } structTypeName.append(">"); assertEquals(structTypeName.toString(), soi1.getTypeName()); - + // null assertNull(soi1.getStructFieldData(null, fields.get(0))); assertNull(soi1.getStructFieldData(null, fields.get(1))); assertNull(soi1.getStructFieldData(null, fields.get(2))); assertNull(soi1.getStructFieldsDataAsList(null)); - + // HashStruct ArrayList struct = new ArrayList(3); struct.add(1); struct.add("two"); struct.add(true); - + assertEquals(1, soi1.getStructFieldData(struct, fields.get(0))); assertEquals("two", soi1.getStructFieldData(struct, fields.get(1))); assertEquals(true, soi1.getStructFieldData(struct, fields.get(2))); - + // Settable Object struct3 = soi1.create(); System.out.println(struct3); @@ -292,12 +322,12 @@ soi1.setStructFieldData(struct3, fields.get(1), "two"); soi1.setStructFieldData(struct3, fields.get(2), true); assertEquals(struct, struct3); - + } catch (Throwable e) { e.printStackTrace(); throw e; } - + } - + } Index: serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java (working copy) @@ -22,47 +22,49 @@ import java.util.List; import junit.framework.TestCase; -import org.apache.hadoop.hive.serde2.thrift.test.Complex; -import org.apache.hadoop.hive.serde2.thrift.test.IntString; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.thrift.test.Complex; +import org.apache.hadoop.hive.serde2.thrift.test.IntString; public class TestThriftObjectInspectors extends TestCase { public void testThriftObjectInspectors() throws Throwable { try { - ObjectInspector oi1 = ObjectInspectorFactory.getReflectionObjectInspector( - Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT); - ObjectInspector oi2 = ObjectInspectorFactory.getReflectionObjectInspector( - Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT); + ObjectInspector oi1 = ObjectInspectorFactory + .getReflectionObjectInspector(Complex.class, + ObjectInspectorFactory.ObjectInspectorOptions.THRIFT); + ObjectInspector oi2 = ObjectInspectorFactory + .getReflectionObjectInspector(Complex.class, + ObjectInspectorFactory.ObjectInspectorOptions.THRIFT); assertEquals(oi1, oi2); - + // metadata assertEquals(Category.STRUCT, oi1.getCategory()); - StructObjectInspector soi = (StructObjectInspector)oi1; + StructObjectInspector soi = (StructObjectInspector) oi1; List fields = soi.getAllStructFieldRefs(); assertEquals(6, fields.size()); assertEquals(fields.get(0), soi.getStructFieldRef("aint")); - + // null - for (int i=0; i c2 = Arrays.asList(new Integer[]{1,2,3}); - c.lint = c2; - List c3 = Arrays.asList(new String[]{"one", "two"}); - c.lString = c3; - List c4 = new ArrayList(); + List c2 = Arrays.asList(new Integer[] { 1, 2, 3 }); + c.lint = c2; + List c3 = Arrays.asList(new String[] { "one", "two" }); + c.lString = c3; + List c4 = new ArrayList(); c.lintString = c4; - c.mStringString = null; - + c.mStringString = null; + assertEquals(1, soi.getStructFieldData(c, fields.get(0))); assertEquals("test", soi.getStructFieldData(c, fields.get(1))); assertEquals(c2, soi.getStructFieldData(c, fields.get(2))); @@ -70,30 +72,33 @@ assertEquals(c4, soi.getStructFieldData(c, fields.get(4))); assertNull(soi.getStructFieldData(c, fields.get(5))); ArrayList cfields = new ArrayList(); - for(int i=0; i<6; i++) { + for (int i = 0; i < 6; i++) { cfields.add(soi.getStructFieldData(c, fields.get(i))); } assertEquals(cfields, soi.getStructFieldsDataAsList(c)); - + // sub fields assertEquals(PrimitiveObjectInspectorFactory.javaIntObjectInspector, fields.get(0).getFieldObjectInspector()); assertEquals(PrimitiveObjectInspectorFactory.javaStringObjectInspector, fields.get(1).getFieldObjectInspector()); - assertEquals(ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.javaIntObjectInspector), + assertEquals( + ObjectInspectorFactory + .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector), fields.get(2).getFieldObjectInspector()); - assertEquals(ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.javaStringObjectInspector), + assertEquals( + ObjectInspectorFactory + .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields.get(3).getFieldObjectInspector()); - assertEquals(ObjectInspectorFactory.getStandardListObjectInspector( - ObjectInspectorFactory.getReflectionObjectInspector(IntString.class, - ObjectInspectorFactory.ObjectInspectorOptions.THRIFT)), + assertEquals(ObjectInspectorFactory + .getStandardListObjectInspector(ObjectInspectorFactory + .getReflectionObjectInspector(IntString.class, + ObjectInspectorFactory.ObjectInspectorOptions.THRIFT)), fields.get(4).getFieldObjectInspector()); assertEquals(ObjectInspectorFactory.getStandardMapObjectInspector( PrimitiveObjectInspectorFactory.javaStringObjectInspector, - PrimitiveObjectInspectorFactory.javaStringObjectInspector), - fields.get(5).getFieldObjectInspector()); + PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields + .get(5).getFieldObjectInspector()); } catch (Throwable e) { e.printStackTrace(); throw e; Index: serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java (working copy) @@ -17,26 +17,19 @@ */ package org.apache.hadoop.hive.serde2.objectinspector; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import junit.framework.TestCase; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.thrift.test.Complex; -import org.apache.hadoop.hive.serde2.thrift.test.IntString; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.FloatWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; -import junit.framework.TestCase; - public class TestObjectInspectorConverters extends TestCase { public void testObjectInspectorConverters() throws Throwable { @@ -45,110 +38,78 @@ Converter booleanConverter = ObjectInspectorConverters.getConverter( PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableBooleanObjectInspector); - assertEquals("BooleanConverter", - new BooleanWritable(false), - booleanConverter.convert(Integer.valueOf(0)) - ); - assertEquals("BooleanConverter", - new BooleanWritable(true), - booleanConverter.convert(Integer.valueOf(1)) - ); - + assertEquals("BooleanConverter", new BooleanWritable(false), + booleanConverter.convert(Integer.valueOf(0))); + assertEquals("BooleanConverter", new BooleanWritable(true), + booleanConverter.convert(Integer.valueOf(1))); + // Byte Converter byteConverter = ObjectInspectorConverters.getConverter( PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableByteObjectInspector); - assertEquals("ByteConverter", - new ByteWritable((byte)0), - byteConverter.convert(Integer.valueOf(0)) - ); - assertEquals("ByteConverter", - new ByteWritable((byte)1), - byteConverter.convert(Integer.valueOf(1)) - ); - + assertEquals("ByteConverter", new ByteWritable((byte) 0), byteConverter + .convert(Integer.valueOf(0))); + assertEquals("ByteConverter", new ByteWritable((byte) 1), byteConverter + .convert(Integer.valueOf(1))); + // Short Converter shortConverter = ObjectInspectorConverters.getConverter( PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableShortObjectInspector); - assertEquals("ShortConverter", - new ShortWritable((short)0), - shortConverter.convert(Integer.valueOf(0)) - ); - assertEquals("ShortConverter", - new ShortWritable((short)1), - shortConverter.convert(Integer.valueOf(1)) - ); - + assertEquals("ShortConverter", new ShortWritable((short) 0), + shortConverter.convert(Integer.valueOf(0))); + assertEquals("ShortConverter", new ShortWritable((short) 1), + shortConverter.convert(Integer.valueOf(1))); + // Int Converter intConverter = ObjectInspectorConverters.getConverter( PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableIntObjectInspector); - assertEquals("IntConverter", - new IntWritable((int)0), - intConverter.convert(Integer.valueOf(0)) - ); - assertEquals("IntConverter", - new IntWritable((int)1), - intConverter.convert(Integer.valueOf(1)) - ); - + assertEquals("IntConverter", new IntWritable(0), intConverter + .convert(Integer.valueOf(0))); + assertEquals("IntConverter", new IntWritable(1), intConverter + .convert(Integer.valueOf(1))); + // Long Converter longConverter = ObjectInspectorConverters.getConverter( PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableLongObjectInspector); - assertEquals("LongConverter", - new LongWritable((long)0), - longConverter.convert(Integer.valueOf(0)) - ); - assertEquals("LongConverter", - new LongWritable((long)1), - longConverter.convert(Integer.valueOf(1)) - ); - + assertEquals("LongConverter", new LongWritable(0), longConverter + .convert(Integer.valueOf(0))); + assertEquals("LongConverter", new LongWritable(1), longConverter + .convert(Integer.valueOf(1))); + // Float Converter floatConverter = ObjectInspectorConverters.getConverter( PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableFloatObjectInspector); - assertEquals("LongConverter", - new FloatWritable((float)0), - floatConverter.convert(Integer.valueOf(0)) - ); - assertEquals("LongConverter", - new FloatWritable((float)1), - floatConverter.convert(Integer.valueOf(1)) - ); - + assertEquals("LongConverter", new FloatWritable(0), floatConverter + .convert(Integer.valueOf(0))); + assertEquals("LongConverter", new FloatWritable(1), floatConverter + .convert(Integer.valueOf(1))); + // Double Converter doubleConverter = ObjectInspectorConverters.getConverter( PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); - assertEquals("DoubleConverter", - new DoubleWritable((double)0), - doubleConverter.convert(Integer.valueOf(0)) - ); - assertEquals("DoubleConverter", - new DoubleWritable((double)1), - doubleConverter.convert(Integer.valueOf(1)) - ); - + assertEquals("DoubleConverter", new DoubleWritable(0), doubleConverter + .convert(Integer.valueOf(0))); + assertEquals("DoubleConverter", new DoubleWritable(1), doubleConverter + .convert(Integer.valueOf(1))); + // Text Converter textConverter = ObjectInspectorConverters.getConverter( PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableStringObjectInspector); - assertEquals("TextConverter", - new Text("0"), - textConverter.convert(Integer.valueOf(0)) - ); - assertEquals("TextConverter", - new Text("1"), - textConverter.convert(Integer.valueOf(1)) - ); - + assertEquals("TextConverter", new Text("0"), textConverter + .convert(Integer.valueOf(0))); + assertEquals("TextConverter", new Text("1"), textConverter + .convert(Integer.valueOf(1))); + } catch (Throwable e) { e.printStackTrace(); throw e; } - - } + + } } Index: serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/MyStruct.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/MyStruct.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/MyStruct.java (working copy) @@ -20,16 +20,15 @@ import java.util.List; import java.util.Map; - /** * Class for testing TestReflectionObjectInspector. */ public class MyStruct { - public int myInt; - public Integer myInteger; - public String myString; - // Note: this is a recursive struct - public MyStruct myStruct; - public List myListString; - public Map myMapStringString; + public int myInt; + public Integer myInteger; + public String myString; + // Note: this is a recursive struct + public MyStruct myStruct; + public List myListString; + public Map myMapStringString; } Index: serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestReflectionObjectInspectors.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestReflectionObjectInspectors.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestReflectionObjectInspectors.java (working copy) @@ -21,57 +21,60 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import junit.framework.TestCase; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; public class TestReflectionObjectInspectors extends TestCase { public void testReflectionObjectInspectors() throws Throwable { try { - ObjectInspector oi1 = ObjectInspectorFactory.getReflectionObjectInspector( - MyStruct.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); - ObjectInspector oi2 = ObjectInspectorFactory.getReflectionObjectInspector( - MyStruct.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + ObjectInspector oi1 = ObjectInspectorFactory + .getReflectionObjectInspector(MyStruct.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + ObjectInspector oi2 = ObjectInspectorFactory + .getReflectionObjectInspector(MyStruct.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); assertEquals(oi1, oi2); - + // metadata assertEquals(Category.STRUCT, oi1.getCategory()); - StructObjectInspector soi = (StructObjectInspector)oi1; - List fields = soi.getAllStructFieldRefs(); + StructObjectInspector soi = (StructObjectInspector) oi1; + List fields = soi.getAllStructFieldRefs(); assertEquals(6, fields.size()); assertEquals(fields.get(2), soi.getStructFieldRef("myString")); - + // null - for (int i=0; i(); a.myMapStringString.put("key", "value"); - + assertEquals(1, soi.getStructFieldData(a, fields.get(0))); assertEquals(2, soi.getStructFieldData(a, fields.get(1))); assertEquals("test", soi.getStructFieldData(a, fields.get(2))); assertEquals(a, soi.getStructFieldData(a, fields.get(3))); assertEquals(a.myListString, soi.getStructFieldData(a, fields.get(4))); - assertEquals(a.myMapStringString, soi.getStructFieldData(a, fields.get(5))); + assertEquals(a.myMapStringString, soi + .getStructFieldData(a, fields.get(5))); ArrayList afields = new ArrayList(); - for(int i=0; i<6; i++) { + for (int i = 0; i < 6; i++) { afields.add(soi.getStructFieldData(a, fields.get(i))); } assertEquals(afields, soi.getStructFieldsDataAsList(a)); - + // sub fields assertEquals(PrimitiveObjectInspectorFactory.javaIntObjectInspector, fields.get(0).getFieldObjectInspector()); @@ -80,13 +83,14 @@ assertEquals(PrimitiveObjectInspectorFactory.javaStringObjectInspector, fields.get(2).getFieldObjectInspector()); assertEquals(soi, fields.get(3).getFieldObjectInspector()); - assertEquals(ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.javaStringObjectInspector), + assertEquals( + ObjectInspectorFactory + .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields.get(4).getFieldObjectInspector()); assertEquals(ObjectInspectorFactory.getStandardMapObjectInspector( PrimitiveObjectInspectorFactory.javaStringObjectInspector, - PrimitiveObjectInspectorFactory.javaStringObjectInspector), - fields.get(5).getFieldObjectInspector()); + PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields + .get(5).getFieldObjectInspector()); } catch (Throwable e) { e.printStackTrace(); throw e; Index: serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestUnionStructObjectInspector.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestUnionStructObjectInspector.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestUnionStructObjectInspector.java (working copy) @@ -21,14 +21,13 @@ import java.util.Arrays; import java.util.List; +import junit.framework.TestCase; + import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import junit.framework.TestCase; - public class TestUnionStructObjectInspector extends TestCase { - public void testUnionStructObjectInspector() throws Throwable { try { ArrayList fieldNames1 = new ArrayList(); @@ -36,49 +35,60 @@ fieldNames1.add("secondString"); fieldNames1.add("thirdBoolean"); ArrayList fieldObjectInspectors1 = new ArrayList(); - fieldObjectInspectors1.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector); - fieldObjectInspectors1.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); - fieldObjectInspectors1.add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector); - StandardStructObjectInspector soi1 = ObjectInspectorFactory.getStandardStructObjectInspector( - fieldNames1, fieldObjectInspectors1); - + fieldObjectInspectors1 + .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector); + fieldObjectInspectors1 + .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); + fieldObjectInspectors1 + .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector); + StandardStructObjectInspector soi1 = ObjectInspectorFactory + .getStandardStructObjectInspector(fieldNames1, fieldObjectInspectors1); + ArrayList fieldNames2 = new ArrayList(); fieldNames2.add("fourthDouble"); fieldNames2.add("fifthLong"); ArrayList fieldObjectInspectors2 = new ArrayList(); - fieldObjectInspectors2.add(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector); - fieldObjectInspectors2.add(PrimitiveObjectInspectorFactory.javaLongObjectInspector); - StandardStructObjectInspector soi2 = ObjectInspectorFactory.getStandardStructObjectInspector( - fieldNames2, fieldObjectInspectors2); - - UnionStructObjectInspector usoi1 = ObjectInspectorFactory.getUnionStructObjectInspector( - Arrays.asList(new StructObjectInspector[]{soi1, soi2})); - UnionStructObjectInspector usoi2 = ObjectInspectorFactory.getUnionStructObjectInspector( - Arrays.asList(new StructObjectInspector[]{soi1, soi2})); - + fieldObjectInspectors2 + .add(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector); + fieldObjectInspectors2 + .add(PrimitiveObjectInspectorFactory.javaLongObjectInspector); + StandardStructObjectInspector soi2 = ObjectInspectorFactory + .getStandardStructObjectInspector(fieldNames2, fieldObjectInspectors2); + + UnionStructObjectInspector usoi1 = ObjectInspectorFactory + .getUnionStructObjectInspector(Arrays + .asList(new StructObjectInspector[] { soi1, soi2 })); + UnionStructObjectInspector usoi2 = ObjectInspectorFactory + .getUnionStructObjectInspector(Arrays + .asList(new StructObjectInspector[] { soi1, soi2 })); + assertEquals(usoi1, usoi2); - + // metadata assertEquals(Category.STRUCT, usoi1.getCategory()); List fields = usoi1.getAllStructFieldRefs(); assertEquals(5, fields.size()); - for (int i=0; i<5; i++) { - if (i<=2) { - assertEquals(fieldNames1.get(i).toLowerCase(), fields.get(i).getFieldName()); - assertEquals(fieldObjectInspectors1.get(i), fields.get(i).getFieldObjectInspector()); + for (int i = 0; i < 5; i++) { + if (i <= 2) { + assertEquals(fieldNames1.get(i).toLowerCase(), fields.get(i) + .getFieldName()); + assertEquals(fieldObjectInspectors1.get(i), fields.get(i) + .getFieldObjectInspector()); } else { - assertEquals(fieldNames2.get(i-3).toLowerCase(), fields.get(i).getFieldName()); - assertEquals(fieldObjectInspectors2.get(i-3), fields.get(i).getFieldObjectInspector()); + assertEquals(fieldNames2.get(i - 3).toLowerCase(), fields.get(i) + .getFieldName()); + assertEquals(fieldObjectInspectors2.get(i - 3), fields.get(i) + .getFieldObjectInspector()); } } assertEquals(fields.get(1), usoi1.getStructFieldRef("secondString")); assertEquals(fields.get(4), usoi1.getStructFieldRef("fifthLong")); - + // null - for (int i=0; i<5; i++) { + for (int i = 0; i < 5; i++) { assertNull(usoi1.getStructFieldData(null, fields.get(i))); } - + // real struct ArrayList struct1 = new ArrayList(3); struct1.add(1); @@ -93,14 +103,15 @@ ArrayList all = new ArrayList(5); all.addAll(struct1); all.addAll(struct2); - - for (int i=0; i<5; i++) { - assertEquals(all.get(i), usoi1.getStructFieldData(struct, fields.get(i))); + + for (int i = 0; i < 5; i++) { + assertEquals(all.get(i), usoi1 + .getStructFieldData(struct, fields.get(i))); } } catch (Throwable e) { e.printStackTrace(); throw e; } } - + } Index: serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java (revision 901519) +++ serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java (working copy) @@ -21,47 +21,49 @@ import java.util.Arrays; import java.util.List; +import junit.framework.TestCase; + import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.thrift.test.Complex; import org.apache.hadoop.hive.serde2.thrift.test.IntString; -import junit.framework.TestCase; - public class TestObjectInspectorUtils extends TestCase { public void testObjectInspectorUtils() throws Throwable { try { - ObjectInspector oi1 = ObjectInspectorFactory.getReflectionObjectInspector( - Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT); - + ObjectInspector oi1 = ObjectInspectorFactory + .getReflectionObjectInspector(Complex.class, + ObjectInspectorFactory.ObjectInspectorOptions.THRIFT); + // metadata assertEquals(Category.STRUCT, oi1.getCategory()); // standard ObjectInspector - StructObjectInspector soi = (StructObjectInspector)ObjectInspectorUtils.getStandardObjectInspector(oi1); + StructObjectInspector soi = (StructObjectInspector) ObjectInspectorUtils + .getStandardObjectInspector(oi1); List fields = soi.getAllStructFieldRefs(); assertEquals(6, fields.size()); assertEquals(fields.get(0), soi.getStructFieldRef("aint")); - + // null - for (int i=0; i c2 = Arrays.asList(new Integer[]{1,2,3}); - cc.lint = c2; - List c3 = Arrays.asList(new String[]{"one", "two"}); - cc.lString = c3; - List c4 = new ArrayList(); + List c2 = Arrays.asList(new Integer[] { 1, 2, 3 }); + cc.lint = c2; + List c3 = Arrays.asList(new String[] { "one", "two" }); + cc.lString = c3; + List c4 = new ArrayList(); cc.lintString = c4; - cc.mStringString = null; + cc.mStringString = null; // standard object Object c = ObjectInspectorUtils.copyToStandardObject(cc, oi1); - + assertEquals(1, soi.getStructFieldData(c, fields.get(0))); assertEquals("test", soi.getStructFieldData(c, fields.get(1))); assertEquals(c2, soi.getStructFieldData(c, fields.get(2))); @@ -69,35 +71,38 @@ assertEquals(c4, soi.getStructFieldData(c, fields.get(4))); assertNull(soi.getStructFieldData(c, fields.get(5))); ArrayList cfields = new ArrayList(); - for(int i=0; i<6; i++) { + for (int i = 0; i < 6; i++) { cfields.add(soi.getStructFieldData(c, fields.get(i))); } assertEquals(cfields, soi.getStructFieldsDataAsList(c)); - + // sub fields assertEquals(PrimitiveObjectInspectorFactory.javaIntObjectInspector, fields.get(0).getFieldObjectInspector()); assertEquals(PrimitiveObjectInspectorFactory.javaStringObjectInspector, fields.get(1).getFieldObjectInspector()); - assertEquals(ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.javaIntObjectInspector), + assertEquals( + ObjectInspectorFactory + .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector), fields.get(2).getFieldObjectInspector()); - assertEquals(ObjectInspectorFactory.getStandardListObjectInspector( - PrimitiveObjectInspectorFactory.javaStringObjectInspector), + assertEquals( + ObjectInspectorFactory + .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields.get(3).getFieldObjectInspector()); - assertEquals(ObjectInspectorUtils.getStandardObjectInspector( - ObjectInspectorFactory.getStandardListObjectInspector( - ObjectInspectorFactory.getReflectionObjectInspector(IntString.class, - ObjectInspectorFactory.ObjectInspectorOptions.THRIFT))), + assertEquals(ObjectInspectorUtils + .getStandardObjectInspector(ObjectInspectorFactory + .getStandardListObjectInspector(ObjectInspectorFactory + .getReflectionObjectInspector(IntString.class, + ObjectInspectorFactory.ObjectInspectorOptions.THRIFT))), fields.get(4).getFieldObjectInspector()); assertEquals(ObjectInspectorFactory.getStandardMapObjectInspector( PrimitiveObjectInspectorFactory.javaStringObjectInspector, - PrimitiveObjectInspectorFactory.javaStringObjectInspector), - fields.get(5).getFieldObjectInspector()); + PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields + .get(5).getFieldObjectInspector()); } catch (Throwable e) { e.printStackTrace(); throw e; } - - } + + } }