diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java index b689336..82816f7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java @@ -36,14 +36,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; @@ -55,7 +47,7 @@ import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; - +import parquet.io.ParquetEncodingException; import parquet.io.api.Binary; /** @@ -125,6 +117,16 @@ public Object deserialize(final Writable blob) throws SerDeException { deserializedSize = 0; if (blob instanceof ArrayWritable) { deserializedSize = ((ArrayWritable) blob).get().length; + //in order to support changing column type, we would have to + //inspect each element of array writable. + Writable[] values = ((ArrayWritable) blob).get(); + final List fields = ((StructObjectInspector)objInspector).getAllStructFieldRefs(); + for (int i=0; i:array"); + serDe.initialize(conf, tbl); - final ArrayWritable arrWritable = new ArrayWritable(Writable.class, arr); // Test deserializeAndSerializeLazySimple(serDe, arrWritable); - System.out.println("test: testParquetHiveSerDe - OK"); + } catch (final Throwable e) { + e.printStackTrace(); + throw e; + } + } + + public void testParquetHiveSerDe3() throws Throwable { + try { + final ParquetHiveSerDe serDe = new ParquetHiveSerDe(); + final Configuration conf = new Configuration(); + final Properties tbl = createProperties(); + //change the string type to be int type, after deserialize, the column would contain null value + //change the bigint type to be int type, after deserialize, the column contains the same numerical value + tbl.setProperty("columns.types", "tinyint:smallint:int:int:double:int:map:array"); + serDe.initialize(conf, tbl); + + // Test + final Object row = serDe.deserialize(arrWritable); + final Writable[] deserized = ((ArrayWritable)row).get(); + assertNull(deserized[5]); + assertEquals(arr[3], deserized[3]); } catch (final Throwable e) { e.printStackTrace();