diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java index 0a2f44c..4802473 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java @@ -19,16 +19,12 @@ package org.apache.hadoop.hive.serde2.lazy; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; -import org.apache.commons.lang.ArrayUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.AbstractEncodingAwareSerDe; import org.apache.hadoop.hive.serde2.ByteStream; @@ -36,7 +32,6 @@ import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.SerDeUtils; -import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; @@ -48,14 +43,10 @@ import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.io.BinaryComparable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; -import org.apache.hive.common.util.HiveStringUtils; - /** * LazySimpleSerDe can be used to read the same data format as @@ -413,4 +404,29 @@ protected Writable transformToUTF8(Writable blob) { Text text = (Text)blob; return SerDeUtils.transformTextToUTF8(text, this.charset); } + + /** + * This method is deprecated and is only used for backward compatibility. + * Replaced by @see org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters#LazySerDeParameters(Configuration, Properties, String) + */ + @Deprecated + public static SerDeParameters initSerdeParams(Configuration job, + Properties tbl, + String serdeName) throws SerDeException { + return new SerDeParameters(job, tbl, serdeName); + } + + /** + * This class is deprecated and is only used for backward compatibility. Replace by + * @see org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters. + */ + @Deprecated + public static class SerDeParameters extends org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters { + + public SerDeParameters(Configuration job, + Properties tbl, + String serdeName) throws SerDeException { + super(job, tbl, serdeName); + } + } } diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java index b11ce32..391edd4 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.List; import java.util.Properties; +import java.util.Random; import junit.framework.TestCase; @@ -29,15 +30,22 @@ import org.apache.hadoop.hive.serde2.ByteStream; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeUtils; +import org.apache.hadoop.hive.serde2.binarysortable.MyTestClass; +import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; +import org.junit.Test; /** * TestLazySimpleSerDe. @@ -81,8 +89,6 @@ public void testLazySimpleSerDe() throws Throwable { } } - - /** * Test the LazySimpleSerDe class with LastColumnTakesRest option. */ @@ -167,20 +173,56 @@ public void testLazySimpleSerDeMissingColumns() throws Throwable { throw e; } } - - Object serializeAndDeserialize(List o1, StructObjectInspector oi1, + + /** + * Tests the deprecated usage of SerDeParameters. + * + */ + @Test + @SuppressWarnings("deprecation") + public void testSerDeParameters() throws SerDeException, IOException { + // Setup + LazySimpleSerDe serDe = new LazySimpleSerDe(); + Configuration conf = new Configuration(); + + MyTestClass row = new MyTestClass(); + ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo(); + row.randomFill(new Random(1234), extraTypeInfo); + + StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory + .getReflectionObjectInspector(MyTestClass.class, + ObjectInspectorOptions.JAVA); + + String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI); + String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI); + + Properties schema = new Properties(); + schema.setProperty(serdeConstants.LIST_COLUMNS, fieldNames); + schema.setProperty(serdeConstants.LIST_COLUMN_TYPES, fieldTypes); + + SerDeUtils.initializeSerDe(serDe, conf, schema, null); + SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, schema, "testSerdeName"); + + // Test + LazyStruct data = (LazyStruct)serializeAndDeserialize(row, rowOI, serDe, serdeParams); + assertEquals((boolean)row.myBool, ((LazyBoolean)data.getField(0)).getWritableObject().get()); + assertEquals((int)row.myInt, ((LazyInteger)data.getField(3)).getWritableObject().get()); + } + + private Object serializeAndDeserialize(Object row, + StructObjectInspector rowOI, LazySimpleSerDe serde, LazySerDeParameters serdeParams) throws IOException, SerDeException { ByteStream.Output serializeStream = new ByteStream.Output(); - LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams + LazySimpleSerDe.serialize(serializeStream, row, rowOI, serdeParams .getSeparators(), 0, serdeParams.getNullSequence(), serdeParams .isEscaped(), serdeParams.getEscapeChar(), serdeParams .getNeedsEscape()); + Text t = new Text(serializeStream.toByteArray()); return serde.deserialize(t); } - - + private void deserializeAndSerialize(LazySimpleSerDe serDe, Text t, String s, Object[] expectedFieldsData) throws SerDeException { // Get the row structure