diff --git data/files/nested_complex.txt data/files/nested_complex.txt new file mode 100644 index 0000000..fee462c --- /dev/null +++ data/files/nested_complex.txt @@ -0,0 +1,2 @@ +3012k1v1k2v2a102 +2032k1v1k3v3b102 diff --git hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java index 3bd0919..db69ae5 100644 --- hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java +++ hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java @@ -57,8 +57,9 @@ public class TestLazyHBaseObject extends TestCase { /** * Test the LazyMap class with Integer-to-String. + * @throws SerDeException */ - public void testLazyHBaseCellMap1() { + public void testLazyHBaseCellMap1() throws SerDeException { // Map of Integer to String Text nullSequence = new Text("\\N"); ObjectInspector oi = LazyFactory.createLazyObjectInspector( @@ -118,8 +119,9 @@ public void testLazyHBaseCellMap1() { /** * Test the LazyMap class with String-to-String. + * @throws SerDeException */ - public void testLazyHBaseCellMap2() { + public void testLazyHBaseCellMap2() throws SerDeException { // Map of String to String Text nullSequence = new Text("\\N"); ObjectInspector oi = LazyFactory.createLazyObjectInspector( @@ -180,8 +182,9 @@ public void testLazyHBaseCellMap2() { /** * Test the LazyHBaseCellMap class for the case where both the key and the value in the family * map are stored in binary format using the appropriate LazyPrimitive objects. + * @throws SerDeException */ - public void testLazyHBaseCellMap3() { + public void testLazyHBaseCellMap3() throws SerDeException { Text nullSequence = new Text("\\N"); TypeInfo mapBinaryIntKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map"); @@ -450,8 +453,9 @@ public void testLazyHBaseCellMap3() { /** * Test the LazyHBaseRow class with one-for-one mappings between * Hive fields and HBase columns. + * @throws SerDeException */ - public void testLazyHBaseRow1() { + public void testLazyHBaseRow1() throws SerDeException { List fieldTypeInfos = TypeInfoUtils.getTypeInfosFromTypeString( "string,int,array,map,string"); @@ -573,8 +577,9 @@ public void testLazyHBaseRow1() { /** * Test the LazyHBaseRow class with a mapping from a Hive field to * an HBase column family. + * @throws SerDeException */ - public void testLazyHBaseRow2() { + public void testLazyHBaseRow2() throws SerDeException { // column family is mapped to Map List fieldTypeInfos = TypeInfoUtils.getTypeInfosFromTypeString( @@ -695,8 +700,9 @@ public void testLazyHBaseRow2() { * Test the LazyHBaseRow class with a one-to-one/onto mapping between Hive columns and * HBase column family/column qualifier pairs. The column types are primitive and fields * are stored in binary format in HBase. + * @throws SerDeException */ - public void testLazyHBaseRow3() { + public void testLazyHBaseRow3() throws SerDeException { List fieldTypeInfos = TypeInfoUtils.getTypeInfosFromTypeString( "string,int,tinyint,smallint,bigint,float,double,string,boolean"); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index 04921d5..89964f0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@ -270,7 +270,11 @@ public static TableDesc getDefaultQueryOutputTableDesc(String cols, String colTy String fileFormat) { TableDesc tblDesc = getTableDesc(LazySimpleSerDe.class, "" + Utilities.ctrlaCode, cols, colTypes, false, false, fileFormat); + //enable escaping tblDesc.getProperties().setProperty(serdeConstants.ESCAPE_CHAR, "\\"); + //enable extended nesting levels + tblDesc.getProperties().setProperty( + LazySimpleSerDe.SERIALIZATION_EXTEND_NESTING_LEVELS, "true"); return tblDesc; } diff --git ql/src/test/queries/clientnegative/nested_complex_neg.q ql/src/test/queries/clientnegative/nested_complex_neg.q new file mode 100644 index 0000000..ac6c4ee --- /dev/null +++ ql/src/test/queries/clientnegative/nested_complex_neg.q @@ -0,0 +1,15 @@ + +create table nestedcomplex ( +simple_int int, +max_nested_array array>>>>>>>>>>>>>>>>>>>>>>, +max_nested_map array>>>>>>>>>>>>>>>>>>>>>, +max_nested_struct array>>>>>>>>>>>>>>>>>>>>>>, +simple_string string) + +; + + +-- This should fail in as extended nesting levels are not enabled using the serdeproperty hive.serialization.extend.nesting.levels +load data local inpath '../data/files/nested_complex.txt' overwrite into table nestedcomplex; + +select * from nestedcomplex sort by simple_int; diff --git ql/src/test/queries/clientpositive/nested_complex.q ql/src/test/queries/clientpositive/nested_complex.q new file mode 100644 index 0000000..b94fbb7 --- /dev/null +++ ql/src/test/queries/clientpositive/nested_complex.q @@ -0,0 +1,22 @@ + +create table nestedcomplex ( +simple_int int, +max_nested_array array>>>>>>>>>>>>>>>>>>>>>>, +max_nested_map array>>>>>>>>>>>>>>>>>>>>>, +max_nested_struct array>>>>>>>>>>>>>>>>>>>>>>, +simple_string string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ( + 'hive.serialization.extend.nesting.levels'='true', + 'line.delim'='\n' +) +; + +describe nestedcomplex; +describe extended nestedcomplex; + + +load data local inpath '../data/files/nested_complex.txt' overwrite into table nestedcomplex; + +select * from nestedcomplex sort by simple_int; diff --git ql/src/test/results/clientnegative/nested_complex_neg.q.out ql/src/test/results/clientnegative/nested_complex_neg.q.out new file mode 100644 index 0000000..c046299 --- /dev/null +++ ql/src/test/results/clientnegative/nested_complex_neg.q.out @@ -0,0 +1,9 @@ +PREHOOK: query: create table nestedcomplex ( +simple_int int, +max_nested_array array>>>>>>>>>>>>>>>>>>>>>>, +max_nested_map array>>>>>>>>>>>>>>>>>>>>>, +max_nested_struct array>>>>>>>>>>>>>>>>>>>>>>, +simple_string string) +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask diff --git ql/src/test/results/clientpositive/alter_partition_coltype.q.out ql/src/test/results/clientpositive/alter_partition_coltype.q.out index d9c48aa..2720736 100644 --- ql/src/test/results/clientpositive/alter_partition_coltype.q.out +++ ql/src/test/results/clientpositive/alter_partition_coltype.q.out @@ -201,6 +201,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -389,6 +390,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -636,6 +638,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -796,6 +799,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1201,6 +1205,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out index 492be3a..0e8f96b 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out @@ -247,6 +247,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -445,6 +446,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -721,6 +723,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -955,6 +958,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1118,6 +1122,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_11.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_11.q.out index 7ed2448..42e25fa 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_11.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_11.q.out @@ -356,6 +356,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -674,6 +675,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -903,6 +905,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out index 5b49c35..47a8635 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out @@ -491,6 +491,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out index 1b585bf..c39d057 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out @@ -231,6 +231,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -509,6 +510,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -743,6 +745,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -906,6 +909,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out index c5315fb..bd7381f 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out @@ -183,6 +183,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -332,6 +333,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -606,6 +608,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -838,6 +841,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -954,6 +958,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out index a9ab616..6121722 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out @@ -195,6 +195,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -344,6 +345,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -618,6 +620,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -850,6 +853,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -966,6 +970,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out index 7c4558f..c3ea09d 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out @@ -166,6 +166,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -307,6 +308,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -517,6 +519,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -691,6 +694,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -804,6 +808,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out index fc2ffc5..e0cd848 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out @@ -256,6 +256,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -456,6 +457,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -781,6 +783,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1062,6 +1065,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1226,6 +1230,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out index 3df0ca8..924fbad 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out @@ -256,6 +256,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -456,6 +457,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -783,6 +785,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1064,6 +1067,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1228,6 +1232,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucket_map_join_1.q.out ql/src/test/results/clientpositive/bucket_map_join_1.q.out index 56131b0..06a802c 100644 --- ql/src/test/results/clientpositive/bucket_map_join_1.q.out +++ ql/src/test/results/clientpositive/bucket_map_join_1.q.out @@ -184,6 +184,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucket_map_join_2.q.out ql/src/test/results/clientpositive/bucket_map_join_2.q.out index 1e7bea5..27c741f 100644 --- ql/src/test/results/clientpositive/bucket_map_join_2.q.out +++ ql/src/test/results/clientpositive/bucket_map_join_2.q.out @@ -184,6 +184,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucketcontext_1.q.out ql/src/test/results/clientpositive/bucketcontext_1.q.out index 43e34ce..62910fb 100644 --- ql/src/test/results/clientpositive/bucketcontext_1.q.out +++ ql/src/test/results/clientpositive/bucketcontext_1.q.out @@ -273,6 +273,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -471,6 +472,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucketcontext_2.q.out ql/src/test/results/clientpositive/bucketcontext_2.q.out index ab44de5..0857c9d 100644 --- ql/src/test/results/clientpositive/bucketcontext_2.q.out +++ ql/src/test/results/clientpositive/bucketcontext_2.q.out @@ -261,6 +261,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -459,6 +460,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucketcontext_3.q.out ql/src/test/results/clientpositive/bucketcontext_3.q.out index 592765a..69dc2b2 100644 --- ql/src/test/results/clientpositive/bucketcontext_3.q.out +++ ql/src/test/results/clientpositive/bucketcontext_3.q.out @@ -213,6 +213,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -362,6 +363,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucketcontext_4.q.out ql/src/test/results/clientpositive/bucketcontext_4.q.out index 6fc94a7..0d79901 100644 --- ql/src/test/results/clientpositive/bucketcontext_4.q.out +++ ql/src/test/results/clientpositive/bucketcontext_4.q.out @@ -225,6 +225,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -374,6 +375,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucketcontext_5.q.out ql/src/test/results/clientpositive/bucketcontext_5.q.out index 8eb9a71..e5d2db4 100644 --- ql/src/test/results/clientpositive/bucketcontext_5.q.out +++ ql/src/test/results/clientpositive/bucketcontext_5.q.out @@ -196,6 +196,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -337,6 +338,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucketcontext_6.q.out ql/src/test/results/clientpositive/bucketcontext_6.q.out index 8271292..99f19a0 100644 --- ql/src/test/results/clientpositive/bucketcontext_6.q.out +++ ql/src/test/results/clientpositive/bucketcontext_6.q.out @@ -260,6 +260,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -456,6 +457,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucketcontext_7.q.out ql/src/test/results/clientpositive/bucketcontext_7.q.out index db9bb1d..19ea4fa 100644 --- ql/src/test/results/clientpositive/bucketcontext_7.q.out +++ ql/src/test/results/clientpositive/bucketcontext_7.q.out @@ -286,6 +286,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -486,6 +487,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucketcontext_8.q.out ql/src/test/results/clientpositive/bucketcontext_8.q.out index 21b5dc5..9a7aaa0 100644 --- ql/src/test/results/clientpositive/bucketcontext_8.q.out +++ ql/src/test/results/clientpositive/bucketcontext_8.q.out @@ -286,6 +286,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -486,6 +487,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucketmapjoin1.q.out ql/src/test/results/clientpositive/bucketmapjoin1.q.out index 4bbd35f..9f8552a 100644 --- ql/src/test/results/clientpositive/bucketmapjoin1.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin1.q.out @@ -107,6 +107,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types int:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -226,6 +227,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types int:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucketmapjoin10.q.out ql/src/test/results/clientpositive/bucketmapjoin10.q.out index 3466e6d..1a6bc06 100644 --- ql/src/test/results/clientpositive/bucketmapjoin10.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin10.q.out @@ -300,6 +300,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucketmapjoin11.q.out ql/src/test/results/clientpositive/bucketmapjoin11.q.out index 1c12c09..bd9b1fe 100644 --- ql/src/test/results/clientpositive/bucketmapjoin11.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin11.q.out @@ -314,6 +314,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -554,6 +555,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucketmapjoin12.q.out ql/src/test/results/clientpositive/bucketmapjoin12.q.out index abf9783..fc161a9 100644 --- ql/src/test/results/clientpositive/bucketmapjoin12.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin12.q.out @@ -235,6 +235,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -419,6 +420,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucketmapjoin13.q.out ql/src/test/results/clientpositive/bucketmapjoin13.q.out index 870cb35..30d8925 100644 --- ql/src/test/results/clientpositive/bucketmapjoin13.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin13.q.out @@ -280,6 +280,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -487,6 +488,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -705,6 +707,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -925,6 +928,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucketmapjoin7.q.out ql/src/test/results/clientpositive/bucketmapjoin7.q.out index b8ba7c0..8105ba4 100644 --- ql/src/test/results/clientpositive/bucketmapjoin7.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin7.q.out @@ -130,6 +130,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucketmapjoin8.q.out ql/src/test/results/clientpositive/bucketmapjoin8.q.out index 2a5a5d5..92c74a9 100644 --- ql/src/test/results/clientpositive/bucketmapjoin8.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin8.q.out @@ -207,6 +207,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -407,6 +408,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucketmapjoin9.q.out ql/src/test/results/clientpositive/bucketmapjoin9.q.out index c2db270..b7aec66 100644 --- ql/src/test/results/clientpositive/bucketmapjoin9.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin9.q.out @@ -205,6 +205,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -426,6 +427,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/bucketmapjoin_negative3.q.out ql/src/test/results/clientpositive/bucketmapjoin_negative3.q.out index 2230fd1..07dced9 100644 --- ql/src/test/results/clientpositive/bucketmapjoin_negative3.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin_negative3.q.out @@ -193,6 +193,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -342,6 +343,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -485,6 +487,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -626,6 +629,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -767,6 +771,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -908,6 +913,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1049,6 +1055,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1190,6 +1197,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1331,6 +1339,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/columnstats_partlvl.q.out ql/src/test/results/clientpositive/columnstats_partlvl.q.out index 2c32730..9330b25 100644 --- ql/src/test/results/clientpositive/columnstats_partlvl.q.out +++ ql/src/test/results/clientpositive/columnstats_partlvl.q.out @@ -202,6 +202,7 @@ STAGE PLANS: columns _col0 columns.types struct escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -408,6 +409,7 @@ STAGE PLANS: columns _col0 columns.types struct escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/columnstats_tbllvl.q.out ql/src/test/results/clientpositive/columnstats_tbllvl.q.out index 007bc31..ab411fc 100644 --- ql/src/test/results/clientpositive/columnstats_tbllvl.q.out +++ ql/src/test/results/clientpositive/columnstats_tbllvl.q.out @@ -240,6 +240,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types struct:struct:struct escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/combine2_hadoop20.q.out ql/src/test/results/clientpositive/combine2_hadoop20.q.out index 1ef67f4..7edbd87 100644 --- ql/src/test/results/clientpositive/combine2_hadoop20.q.out +++ ql/src/test/results/clientpositive/combine2_hadoop20.q.out @@ -621,6 +621,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/filter_join_breaktask.q.out ql/src/test/results/clientpositive/filter_join_breaktask.q.out index 52bac6a..de80d6e 100644 --- ql/src/test/results/clientpositive/filter_join_breaktask.q.out +++ ql/src/test/results/clientpositive/filter_join_breaktask.q.out @@ -288,6 +288,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/groupby_sort_1.q.out ql/src/test/results/clientpositive/groupby_sort_1.q.out index e6f3a7a..965bb00 100644 --- ql/src/test/results/clientpositive/groupby_sort_1.q.out +++ ql/src/test/results/clientpositive/groupby_sort_1.q.out @@ -4524,6 +4524,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3,_col4 columns.types string:bigint:string:string:bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/groupby_sort_skew_1.q.out ql/src/test/results/clientpositive/groupby_sort_skew_1.q.out index b7ca0ee..10536b2 100644 --- ql/src/test/results/clientpositive/groupby_sort_skew_1.q.out +++ ql/src/test/results/clientpositive/groupby_sort_skew_1.q.out @@ -4934,6 +4934,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3,_col4 columns.types string:bigint:string:string:bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/input23.q.out ql/src/test/results/clientpositive/input23.q.out index f71a43f..76b9a07 100644 --- ql/src/test/results/clientpositive/input23.q.out +++ ql/src/test/results/clientpositive/input23.q.out @@ -144,6 +144,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7 columns.types string:string:string:string:string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/input42.q.out ql/src/test/results/clientpositive/input42.q.out index 67679af..2ccc32c 100644 --- ql/src/test/results/clientpositive/input42.q.out +++ ql/src/test/results/clientpositive/input42.q.out @@ -158,6 +158,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1348,6 +1349,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1916,6 +1918,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/input_part7.q.out ql/src/test/results/clientpositive/input_part7.q.out index 538a742..300173d 100644 --- ql/src/test/results/clientpositive/input_part7.q.out +++ ql/src/test/results/clientpositive/input_part7.q.out @@ -242,6 +242,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/input_part9.q.out ql/src/test/results/clientpositive/input_part9.q.out index 91d1794..e343c9c 100644 --- ql/src/test/results/clientpositive/input_part9.q.out +++ ql/src/test/results/clientpositive/input_part9.q.out @@ -163,6 +163,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/join_filters_overlap.q.out ql/src/test/results/clientpositive/join_filters_overlap.q.out index 4f79d38..0adccba 100644 --- ql/src/test/results/clientpositive/join_filters_overlap.q.out +++ ql/src/test/results/clientpositive/join_filters_overlap.q.out @@ -181,6 +181,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3,_col4,_col5 columns.types int:int:int:int:int:int escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -387,6 +388,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3,_col4,_col5 columns.types int:int:int:int:int:int escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -593,6 +595,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3,_col4,_col5 columns.types int:int:int:int:int:int escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -825,6 +828,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7 columns.types int:int:int:int:int:int:int:int escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1052,6 +1056,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7 columns.types int:int:int:int:int:int:int:int escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/louter_join_ppr.q.out ql/src/test/results/clientpositive/louter_join_ppr.q.out index 32827a3..e8a5def 100644 --- ql/src/test/results/clientpositive/louter_join_ppr.q.out +++ ql/src/test/results/clientpositive/louter_join_ppr.q.out @@ -249,6 +249,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -651,6 +652,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1054,6 +1056,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1363,6 +1366,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/metadataonly1.q.out ql/src/test/results/clientpositive/metadataonly1.q.out index aa6402e..7c0601a 100644 --- ql/src/test/results/clientpositive/metadataonly1.q.out +++ ql/src/test/results/clientpositive/metadataonly1.q.out @@ -65,6 +65,7 @@ STAGE PLANS: columns _col0 columns.types string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -192,6 +193,7 @@ STAGE PLANS: columns _col0 columns.types string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -322,6 +324,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -446,6 +449,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -832,6 +836,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1081,6 +1086,7 @@ STAGE PLANS: columns _col0,_col1 columns.types string:bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1302,6 +1308,7 @@ STAGE PLANS: columns _col0,_col1 columns.types string:bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1468,6 +1475,7 @@ STAGE PLANS: columns _col0 columns.types string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1819,6 +1827,7 @@ STAGE PLANS: columns _col0,_col1 columns.types string:bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/nested_complex.q.out ql/src/test/results/clientpositive/nested_complex.q.out new file mode 100644 index 0000000..0988ccc --- /dev/null +++ ql/src/test/results/clientpositive/nested_complex.q.out @@ -0,0 +1,63 @@ +PREHOOK: query: create table nestedcomplex ( +simple_int int, +max_nested_array array>>>>>>>>>>>>>>>>>>>>>>, +max_nested_map array>>>>>>>>>>>>>>>>>>>>>, +max_nested_struct array>>>>>>>>>>>>>>>>>>>>>>, +simple_string string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ( + 'hive.serialization.extend.nesting.levels'='true', + 'line.delim'='\n' +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table nestedcomplex ( +simple_int int, +max_nested_array array>>>>>>>>>>>>>>>>>>>>>>, +max_nested_map array>>>>>>>>>>>>>>>>>>>>>, +max_nested_struct array>>>>>>>>>>>>>>>>>>>>>>, +simple_string string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ( + 'hive.serialization.extend.nesting.levels'='true', + 'line.delim'='\n' +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@nestedcomplex +PREHOOK: query: describe nestedcomplex +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe nestedcomplex +POSTHOOK: type: DESCTABLE +simple_int int None +max_nested_array array>>>>>>>>>>>>>>>>>>>>>> None +max_nested_map array>>>>>>>>>>>>>>>>>>>>> None +max_nested_struct array>>>>>>>>>>>>>>>>>>>>>> None +simple_string string None +PREHOOK: query: describe extended nestedcomplex +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended nestedcomplex +POSTHOOK: type: DESCTABLE +simple_int int None +max_nested_array array>>>>>>>>>>>>>>>>>>>>>> None +max_nested_map array>>>>>>>>>>>>>>>>>>>>> None +max_nested_struct array>>>>>>>>>>>>>>>>>>>>>> None +simple_string string None + +#### A masked pattern was here #### +PREHOOK: query: load data local inpath '../data/files/nested_complex.txt' overwrite into table nestedcomplex +PREHOOK: type: LOAD +PREHOOK: Output: default@nestedcomplex +POSTHOOK: query: load data local inpath '../data/files/nested_complex.txt' overwrite into table nestedcomplex +POSTHOOK: type: LOAD +POSTHOOK: Output: default@nestedcomplex +PREHOOK: query: select * from nestedcomplex sort by simple_int +PREHOOK: type: QUERY +PREHOOK: Input: default@nestedcomplex +#### A masked pattern was here #### +POSTHOOK: query: select * from nestedcomplex sort by simple_int +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nestedcomplex +#### A masked pattern was here #### +2 [[[[[[[[[[[[[[[[[[[[[[[0,3,2]]]]]]]]]]]]]]]]]]]]]]] [[[[[[[[[[[[[[[[[[[[[{"k1":"v1","k3":"v3"}]]]]]]]]]]]]]]]]]]]]] [[[[[[[[[[[[[[[[[[[[[[{"s":"b","i":10}]]]]]]]]]]]]]]]]]]]]]] 2 +3 [[[[[[[[[[[[[[[[[[[[[[[0,1,2]]]]]]]]]]]]]]]]]]]]]]] [[[[[[[[[[[[[[[[[[[[[{"k1":"v1","k2":"v2"}]]]]]]]]]]]]]]]]]]]]] [[[[[[[[[[[[[[[[[[[[[[{"s":"a","i":10}]]]]]]]]]]]]]]]]]]]]]] 2 diff --git ql/src/test/results/clientpositive/outer_join_ppr.q.out ql/src/test/results/clientpositive/outer_join_ppr.q.out index f311cce..5c79a4a 100644 --- ql/src/test/results/clientpositive/outer_join_ppr.q.out +++ ql/src/test/results/clientpositive/outer_join_ppr.q.out @@ -338,6 +338,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -731,6 +732,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/pcr.q.out ql/src/test/results/clientpositive/pcr.q.out index cd3caff..b2ca247 100644 --- ql/src/test/results/clientpositive/pcr.q.out +++ ql/src/test/results/clientpositive/pcr.q.out @@ -214,6 +214,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types int:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -457,6 +458,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -699,6 +701,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types int:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -905,6 +908,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types int:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1158,6 +1162,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types int:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1422,6 +1427,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types int:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1645,6 +1651,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1830,6 +1837,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -2100,6 +2108,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -2359,6 +2368,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types int:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -2602,6 +2612,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3,_col4,_col5 columns.types int:string:string:int:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -2925,6 +2936,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3,_col4,_col5 columns.types int:string:string:int:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -3266,6 +3278,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types int:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -3559,6 +3572,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types int:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -4949,6 +4963,7 @@ STAGE PLANS: columns _col0,_col1 columns.types string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -5166,6 +5181,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -5390,6 +5406,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/ppd_join_filter.q.out ql/src/test/results/clientpositive/ppd_join_filter.q.out index d76d5bd..46124f9 100644 --- ql/src/test/results/clientpositive/ppd_join_filter.q.out +++ ql/src/test/results/clientpositive/ppd_join_filter.q.out @@ -283,6 +283,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types string:double:double escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -615,6 +616,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types string:double:double escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -947,6 +949,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types string:double:double escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1279,6 +1282,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types string:double:double escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/ppd_union_view.q.out ql/src/test/results/clientpositive/ppd_union_view.q.out index e3e404e..b64645f 100644 --- ql/src/test/results/clientpositive/ppd_union_view.q.out +++ ql/src/test/results/clientpositive/ppd_union_view.q.out @@ -400,6 +400,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -445,6 +446,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -685,6 +687,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -725,6 +728,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/ppr_allchildsarenull.q.out ql/src/test/results/clientpositive/ppr_allchildsarenull.q.out index 2136a33..f8cce85 100644 --- ql/src/test/results/clientpositive/ppr_allchildsarenull.q.out +++ ql/src/test/results/clientpositive/ppr_allchildsarenull.q.out @@ -56,6 +56,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -247,6 +248,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/rand_partitionpruner1.q.out ql/src/test/results/clientpositive/rand_partitionpruner1.q.out index 2f006c6..47b9447 100644 --- ql/src/test/results/clientpositive/rand_partitionpruner1.q.out +++ ql/src/test/results/clientpositive/rand_partitionpruner1.q.out @@ -44,6 +44,7 @@ STAGE PLANS: columns _col0,_col1 columns.types string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/rand_partitionpruner3.q.out ql/src/test/results/clientpositive/rand_partitionpruner3.q.out index 600a834..a39cdba 100644 --- ql/src/test/results/clientpositive/rand_partitionpruner3.q.out +++ ql/src/test/results/clientpositive/rand_partitionpruner3.q.out @@ -50,6 +50,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -177,6 +178,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/regexp_extract.q.out ql/src/test/results/clientpositive/regexp_extract.q.out index 361d8ed..c666cd6 100644 --- ql/src/test/results/clientpositive/regexp_extract.q.out +++ ql/src/test/results/clientpositive/regexp_extract.q.out @@ -140,6 +140,7 @@ STAGE PLANS: columns _col0,_col1 columns.types string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -398,6 +399,7 @@ STAGE PLANS: columns _col0,_col1 columns.types string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/router_join_ppr.q.out ql/src/test/results/clientpositive/router_join_ppr.q.out index 52d7888..405c43a 100644 --- ql/src/test/results/clientpositive/router_join_ppr.q.out +++ ql/src/test/results/clientpositive/router_join_ppr.q.out @@ -348,6 +348,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -657,6 +658,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -960,6 +962,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1357,6 +1360,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/sample10.q.out ql/src/test/results/clientpositive/sample10.q.out index e4fecbe..b61ef64 100644 --- ql/src/test/results/clientpositive/sample10.q.out +++ ql/src/test/results/clientpositive/sample10.q.out @@ -379,6 +379,7 @@ STAGE PLANS: columns _col0,_col1 columns.types string:bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/sample6.q.out ql/src/test/results/clientpositive/sample6.q.out index cd78d8b..9fa8774 100644 --- ql/src/test/results/clientpositive/sample6.q.out +++ ql/src/test/results/clientpositive/sample6.q.out @@ -677,6 +677,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1051,6 +1052,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1679,6 +1681,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -2150,6 +2153,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -2651,6 +2655,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -2909,6 +2914,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -3042,6 +3048,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/sample8.q.out ql/src/test/results/clientpositive/sample8.q.out index 8f26dc8..792af23 100644 --- ql/src/test/results/clientpositive/sample8.q.out +++ ql/src/test/results/clientpositive/sample8.q.out @@ -361,6 +361,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/sample9.q.out ql/src/test/results/clientpositive/sample9.q.out index 7694961..9a9433a 100644 --- ql/src/test/results/clientpositive/sample9.q.out +++ ql/src/test/results/clientpositive/sample9.q.out @@ -46,6 +46,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/smb_mapjoin9.q.out ql/src/test/results/clientpositive/smb_mapjoin9.q.out index 9a7a793..52be6d0 100644 --- ql/src/test/results/clientpositive/smb_mapjoin9.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin9.q.out @@ -78,6 +78,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types int:string:string:int escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -173,6 +174,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types int:string:string:int escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/smb_mapjoin_13.q.out ql/src/test/results/clientpositive/smb_mapjoin_13.q.out index 1204f88..2bfafd2 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_13.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_13.q.out @@ -191,6 +191,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types int:string:int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -392,6 +393,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types int:string:int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/smb_mapjoin_15.q.out ql/src/test/results/clientpositive/smb_mapjoin_15.q.out index 8990856..fc761ef 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_15.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_15.q.out @@ -165,6 +165,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types int:string:int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -414,6 +415,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3,_col4,_col5 columns.types int:int:string:int:int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -603,6 +605,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3,_col4,_col5 columns.types int:int:string:int:int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -824,6 +827,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3,_col4,_col5 columns.types int:int:string:int:int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out index c390b5e..d484c59 100644 --- ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out +++ ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out @@ -182,6 +182,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out index 7dabb55..f6aae06 100644 --- ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out +++ ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out @@ -214,6 +214,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out index c321351..dbce51a 100644 --- ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out +++ ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out @@ -316,6 +316,7 @@ STAGE PLANS: columns _col0 columns.types bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/transform_ppr1.q.out ql/src/test/results/clientpositive/transform_ppr1.q.out index 740a931..10748c9 100644 --- ql/src/test/results/clientpositive/transform_ppr1.q.out +++ ql/src/test/results/clientpositive/transform_ppr1.q.out @@ -281,6 +281,7 @@ STAGE PLANS: columns _col0,_col1 columns.types string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/transform_ppr2.q.out ql/src/test/results/clientpositive/transform_ppr2.q.out index fb8e039..1a16702 100644 --- ql/src/test/results/clientpositive/transform_ppr2.q.out +++ ql/src/test/results/clientpositive/transform_ppr2.q.out @@ -191,6 +191,7 @@ STAGE PLANS: columns _col0,_col1 columns.types string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/udf_explode.q.out ql/src/test/results/clientpositive/udf_explode.q.out index dc6a513..8809375 100644 --- ql/src/test/results/clientpositive/udf_explode.q.out +++ ql/src/test/results/clientpositive/udf_explode.q.out @@ -48,6 +48,7 @@ STAGE PLANS: columns col columns.types int escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -287,6 +288,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -372,6 +374,7 @@ STAGE PLANS: columns key,value columns.types int:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -625,6 +628,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types int:string:bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/udf_java_method.q.out ql/src/test/results/clientpositive/udf_java_method.q.out index 15e71e6..775dcaf 100644 --- ql/src/test/results/clientpositive/udf_java_method.q.out +++ ql/src/test/results/clientpositive/udf_java_method.q.out @@ -81,6 +81,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3,_col4,_col5,_col6 columns.types string:string:string:string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/udf_reflect.q.out ql/src/test/results/clientpositive/udf_reflect.q.out index 91aeab5..52f252d 100644 --- ql/src/test/results/clientpositive/udf_reflect.q.out +++ ql/src/test/results/clientpositive/udf_reflect.q.out @@ -81,6 +81,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7 columns.types string:string:string:string:string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/udf_reflect2.q.out ql/src/test/results/clientpositive/udf_reflect2.q.out index f2c64cd..5411583 100644 --- ql/src/test/results/clientpositive/udf_reflect2.q.out +++ ql/src/test/results/clientpositive/udf_reflect2.q.out @@ -181,6 +181,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8,_col9,_col10,_col11,_col12,_col13,_col14,_col15,_col16,_col17,_col18,_col19,_col20,_col21,_col22,_col23,_col24,_col25,_col26,_col27,_col28,_col29,_col30 columns.types int:tinyint:smallint:int:bigint:float:double:string:string:string:boolean:boolean:boolean:boolean:boolean:binary:int:int:string:string:string:string:string:timestamp:int:int:int:int:int:int:bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/udtf_explode.q.out ql/src/test/results/clientpositive/udtf_explode.q.out index 2905d44..4c59a5f 100644 --- ql/src/test/results/clientpositive/udtf_explode.q.out +++ ql/src/test/results/clientpositive/udtf_explode.q.out @@ -48,6 +48,7 @@ STAGE PLANS: columns col columns.types int escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -287,6 +288,7 @@ STAGE PLANS: columns _col0,_col1 columns.types int:bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -565,6 +567,7 @@ STAGE PLANS: columns _col0,_col1,_col2 columns.types int:string:bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/union24.q.out ql/src/test/results/clientpositive/union24.q.out index 50ae7e3..e18667e 100644 --- ql/src/test/results/clientpositive/union24.q.out +++ ql/src/test/results/clientpositive/union24.q.out @@ -477,6 +477,7 @@ STAGE PLANS: columns _col0,_col1 columns.types string:bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -965,6 +966,7 @@ STAGE PLANS: columns _col0,_col1 columns.types string:bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false @@ -1518,6 +1520,7 @@ STAGE PLANS: columns _col0,_col1 columns.types string:bigint escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/clientpositive/union_ppr.q.out ql/src/test/results/clientpositive/union_ppr.q.out index 756a9cd..0598ae6 100644 --- ql/src/test/results/clientpositive/union_ppr.q.out +++ ql/src/test/results/clientpositive/union_ppr.q.out @@ -244,6 +244,7 @@ STAGE PLANS: columns _col0,_col1,_col2,_col3 columns.types string:string:string:string escape.delim \ + hive.serialization.extend.nesting.levels true serialization.format 1 TotalFiles: 1 GatherStats: false diff --git ql/src/test/results/compiler/plan/cast1.q.xml ql/src/test/results/compiler/plan/cast1.q.xml index bd40304..7589b60 100644 --- ql/src/test/results/compiler/plan/cast1.q.xml +++ ql/src/test/results/compiler/plan/cast1.q.xml @@ -201,6 +201,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6 diff --git ql/src/test/results/compiler/plan/groupby2.q.xml ql/src/test/results/compiler/plan/groupby2.q.xml index 13cca32..b435021 100755 --- ql/src/test/results/compiler/plan/groupby2.q.xml +++ ql/src/test/results/compiler/plan/groupby2.q.xml @@ -1341,6 +1341,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1,_col2 diff --git ql/src/test/results/compiler/plan/groupby3.q.xml ql/src/test/results/compiler/plan/groupby3.q.xml index 06f0864..b2f76c7 100644 --- ql/src/test/results/compiler/plan/groupby3.q.xml +++ ql/src/test/results/compiler/plan/groupby3.q.xml @@ -1540,6 +1540,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1,_col2,_col3,_col4 diff --git ql/src/test/results/compiler/plan/groupby4.q.xml ql/src/test/results/compiler/plan/groupby4.q.xml index 21deeb9..bbf4ee3 100644 --- ql/src/test/results/compiler/plan/groupby4.q.xml +++ ql/src/test/results/compiler/plan/groupby4.q.xml @@ -987,6 +987,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0 diff --git ql/src/test/results/compiler/plan/groupby5.q.xml ql/src/test/results/compiler/plan/groupby5.q.xml index 521ee86..1a65652 100644 --- ql/src/test/results/compiler/plan/groupby5.q.xml +++ ql/src/test/results/compiler/plan/groupby5.q.xml @@ -1080,6 +1080,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1 diff --git ql/src/test/results/compiler/plan/groupby6.q.xml ql/src/test/results/compiler/plan/groupby6.q.xml index b50d796..08cbb56 100644 --- ql/src/test/results/compiler/plan/groupby6.q.xml +++ ql/src/test/results/compiler/plan/groupby6.q.xml @@ -987,6 +987,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0 diff --git ql/src/test/results/compiler/plan/input20.q.xml ql/src/test/results/compiler/plan/input20.q.xml index 3174490..0159ca5 100644 --- ql/src/test/results/compiler/plan/input20.q.xml +++ ql/src/test/results/compiler/plan/input20.q.xml @@ -1099,6 +1099,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1 diff --git ql/src/test/results/compiler/plan/input8.q.xml ql/src/test/results/compiler/plan/input8.q.xml index cc567d4..91bd148 100644 --- ql/src/test/results/compiler/plan/input8.q.xml +++ ql/src/test/results/compiler/plan/input8.q.xml @@ -197,6 +197,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1,_col2 diff --git ql/src/test/results/compiler/plan/input_part1.q.xml ql/src/test/results/compiler/plan/input_part1.q.xml index ed9d218..18ea3fe 100644 --- ql/src/test/results/compiler/plan/input_part1.q.xml +++ ql/src/test/results/compiler/plan/input_part1.q.xml @@ -238,6 +238,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1,_col2,_col3 diff --git ql/src/test/results/compiler/plan/input_testxpath.q.xml ql/src/test/results/compiler/plan/input_testxpath.q.xml index 58edf34..acfad93 100644 --- ql/src/test/results/compiler/plan/input_testxpath.q.xml +++ ql/src/test/results/compiler/plan/input_testxpath.q.xml @@ -205,6 +205,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1,_col2 diff --git ql/src/test/results/compiler/plan/input_testxpath2.q.xml ql/src/test/results/compiler/plan/input_testxpath2.q.xml index 031b955..2eca8ce 100644 --- ql/src/test/results/compiler/plan/input_testxpath2.q.xml +++ ql/src/test/results/compiler/plan/input_testxpath2.q.xml @@ -209,6 +209,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1,_col2 diff --git ql/src/test/results/compiler/plan/join4.q.xml ql/src/test/results/compiler/plan/join4.q.xml index 391b58d..5a0933f 100644 --- ql/src/test/results/compiler/plan/join4.q.xml +++ ql/src/test/results/compiler/plan/join4.q.xml @@ -1734,6 +1734,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1,_col2,_col3 diff --git ql/src/test/results/compiler/plan/join5.q.xml ql/src/test/results/compiler/plan/join5.q.xml index 2669097..1fdfc26 100644 --- ql/src/test/results/compiler/plan/join5.q.xml +++ ql/src/test/results/compiler/plan/join5.q.xml @@ -1734,6 +1734,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1,_col2,_col3 diff --git ql/src/test/results/compiler/plan/join6.q.xml ql/src/test/results/compiler/plan/join6.q.xml index b92d70b..c1d6666 100644 --- ql/src/test/results/compiler/plan/join6.q.xml +++ ql/src/test/results/compiler/plan/join6.q.xml @@ -1734,6 +1734,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1,_col2,_col3 diff --git ql/src/test/results/compiler/plan/join7.q.xml ql/src/test/results/compiler/plan/join7.q.xml index d6253de..2fd3461 100644 --- ql/src/test/results/compiler/plan/join7.q.xml +++ ql/src/test/results/compiler/plan/join7.q.xml @@ -2468,6 +2468,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1,_col2,_col3,_col4,_col5 diff --git ql/src/test/results/compiler/plan/join8.q.xml ql/src/test/results/compiler/plan/join8.q.xml index e1e71a7..3930860 100644 --- ql/src/test/results/compiler/plan/join8.q.xml +++ ql/src/test/results/compiler/plan/join8.q.xml @@ -1820,6 +1820,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1,_col2,_col3 diff --git ql/src/test/results/compiler/plan/sample1.q.xml ql/src/test/results/compiler/plan/sample1.q.xml index b2c40a3..c193e6b 100644 --- ql/src/test/results/compiler/plan/sample1.q.xml +++ ql/src/test/results/compiler/plan/sample1.q.xml @@ -238,6 +238,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1,_col2,_col3 diff --git ql/src/test/results/compiler/plan/udf1.q.xml ql/src/test/results/compiler/plan/udf1.q.xml index ddc36ec..07fc0a4 100644 --- ql/src/test/results/compiler/plan/udf1.q.xml +++ ql/src/test/results/compiler/plan/udf1.q.xml @@ -201,6 +201,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8,_col9,_col10,_col11,_col12,_col13,_col14,_col15,_col16 diff --git ql/src/test/results/compiler/plan/udf4.q.xml ql/src/test/results/compiler/plan/udf4.q.xml index 8ea82eb..dcb35b9 100644 --- ql/src/test/results/compiler/plan/udf4.q.xml +++ ql/src/test/results/compiler/plan/udf4.q.xml @@ -177,6 +177,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8,_col9,_col10,_col11,_col12,_col13,_col14,_col15,_col16,_col17,_col18 diff --git ql/src/test/results/compiler/plan/udf6.q.xml ql/src/test/results/compiler/plan/udf6.q.xml index 334fe0c..f2dfdbc 100644 --- ql/src/test/results/compiler/plan/udf6.q.xml +++ ql/src/test/results/compiler/plan/udf6.q.xml @@ -197,6 +197,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1 diff --git ql/src/test/results/compiler/plan/udf_case.q.xml ql/src/test/results/compiler/plan/udf_case.q.xml index 67d55b8..70390d6 100644 --- ql/src/test/results/compiler/plan/udf_case.q.xml +++ ql/src/test/results/compiler/plan/udf_case.q.xml @@ -201,6 +201,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1 diff --git ql/src/test/results/compiler/plan/udf_when.q.xml ql/src/test/results/compiler/plan/udf_when.q.xml index 8334326..1447ddd 100644 --- ql/src/test/results/compiler/plan/udf_when.q.xml +++ ql/src/test/results/compiler/plan/udf_when.q.xml @@ -201,6 +201,10 @@ + hive.serialization.extend.nesting.levels + true + + columns _col0,_col1 diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java index 59b1406..f66a767 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyListObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyObjectInspectorFactory; @@ -46,9 +47,9 @@ import org.apache.hadoop.hive.serde2.lazydio.LazyDioLong; import org.apache.hadoop.hive.serde2.lazydio.LazyDioShort; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; @@ -200,10 +201,11 @@ * @param nullSequence * The sequence of bytes representing NULL. * @return The ObjectInspector + * @throws SerDeException */ public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, byte[] separator, int separatorIndex, Text nullSequence, boolean escaped, - byte escapeChar) { + byte escapeChar) throws SerDeException { ObjectInspector.Category c = typeInfo.getCategory(); switch (c) { case PRIMITIVE: @@ -217,13 +219,14 @@ public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, nullSequence, escaped, escapeChar), createLazyObjectInspector( ((MapTypeInfo) typeInfo).getMapValueTypeInfo(), separator, separatorIndex + 2, nullSequence, escaped, escapeChar), - separator[separatorIndex], separator[separatorIndex + 1], + LazyUtils.getSeparator(separator, separatorIndex), + LazyUtils.getSeparator(separator, separatorIndex+1), nullSequence, escaped, escapeChar); case LIST: return LazyObjectInspectorFactory.getLazySimpleListObjectInspector( createLazyObjectInspector(((ListTypeInfo) typeInfo) .getListElementTypeInfo(), separator, separatorIndex + 1, - nullSequence, escaped, escapeChar), separator[separatorIndex], + nullSequence, escaped, escapeChar), LazyUtils.getSeparator(separator, separatorIndex), nullSequence, escaped, escapeChar); case STRUCT: StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; @@ -238,7 +241,8 @@ public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, escapeChar)); } return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector( - fieldNames, fieldObjectInspectors, separator[separatorIndex], + fieldNames, fieldObjectInspectors, + LazyUtils.getSeparator(separator, separatorIndex), nullSequence, false, escaped, escapeChar); case UNION: UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; @@ -249,7 +253,8 @@ public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, escapeChar)); } return LazyObjectInspectorFactory.getLazyUnionObjectInspector(lazyOIs, - separator[separatorIndex], nullSequence, escaped, escapeChar); + LazyUtils.getSeparator(separator, separatorIndex), + nullSequence, escaped, escapeChar); } throw new RuntimeException("Hive LazySerDe Internal error."); @@ -262,13 +267,14 @@ public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, * @param lastColumnTakesRest * whether the last column of the struct should take the rest of the * row if there are extra fields. + * @throws SerDeException * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, * boolean, byte) */ public static ObjectInspector createLazyStructInspector( List columnNames, List typeInfos, byte[] separators, Text nullSequence, boolean lastColumnTakesRest, boolean escaped, - byte escapeChar) { + byte escapeChar) throws SerDeException { ArrayList columnObjectInspectors = new ArrayList( typeInfos.size()); for (int i = 0; i < typeInfos.size(); i++) { @@ -283,13 +289,14 @@ public static ObjectInspector createLazyStructInspector( /** * Create a hierarchical ObjectInspector for ColumnarStruct with the given * columnNames and columnTypeInfos. + * @throws SerDeException * * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, * boolean, byte) */ public static ObjectInspector createColumnarStructInspector( List columnNames, List columnTypes, byte[] separators, - Text nullSequence, boolean escaped, byte escapeChar) { + Text nullSequence, boolean escaped, byte escapeChar) throws SerDeException { ArrayList columnObjectInspectors = new ArrayList( columnTypes.size()); for (int i = 0; i < columnTypes.size(); i++) { diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java index d6b31a6..606208c 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java @@ -65,6 +65,9 @@ public static final Log LOG = LogFactory.getLog(LazySimpleSerDe.class .getName()); + public static final String SERIALIZATION_EXTEND_NESTING_LEVELS + = "hive.serialization.extend.nesting.levels"; + public static final byte[] DefaultSeparators = {(byte) 1, (byte) 2, (byte) 3}; private ObjectInspector cachedObjectInspector; @@ -177,6 +180,7 @@ public byte getEscapeChar() { * * @see SerDe#initialize(Configuration, Properties) */ + @Override public void initialize(Configuration job, Properties tbl) throws SerDeException { @@ -209,9 +213,13 @@ public void initialize(Configuration job, Properties tbl) public static SerDeParameters initSerdeParams(Configuration job, Properties tbl, String serdeName) throws SerDeException { SerDeParameters serdeParams = new SerDeParameters(); - // Read the separators: We use 8 levels of separators by default, but we - // should change this when we allow users to specify more than 10 levels - // of separators through DDL. + // Read the separators: We use 8 levels of separators by default, + // and 24 if SERIALIZATION_EXTEND_NESTING_LEVELS is set to true + // The levels possible are the set of control chars that we can use as + // special delimiters, ie they should absent in the data or escaped. + // To increase this level further, we need to stop relying + // on single control chars delimiters + serdeParams.separators = new byte[8]; serdeParams.separators[0] = getByte(tbl.getProperty(serdeConstants.FIELD_DELIM, tbl.getProperty(serdeConstants.SERIALIZATION_FORMAT)), DefaultSeparators[0]); @@ -219,8 +227,53 @@ public static SerDeParameters initSerdeParams(Configuration job, .getProperty(serdeConstants.COLLECTION_DELIM), DefaultSeparators[1]); serdeParams.separators[2] = getByte( tbl.getProperty(serdeConstants.MAPKEY_DELIM), DefaultSeparators[2]); - for (int i = 3; i < serdeParams.separators.length; i++) { - serdeParams.separators[i] = (byte) (i + 1); + String extendedNesting = + tbl.getProperty(SERIALIZATION_EXTEND_NESTING_LEVELS); + if(extendedNesting == null || !extendedNesting.equalsIgnoreCase("true")){ + //use the default smaller set of separators for backward compatibility + for (int i = 3; i < serdeParams.separators.length; i++) { + serdeParams.separators[i] = (byte) (i + 1); + } + } + else{ + //If extended nesting is enabled, set the extended set of separator chars + + final int MAX_CTRL_CHARS = 29; + byte[] extendedSeparators = new byte[MAX_CTRL_CHARS]; + int extendedSeparatorsIdx = 0; + + //get the first 3 separators that have already been set (defaults to 1,2,3) + for(int i = 0; i < 3; i++){ + extendedSeparators[extendedSeparatorsIdx++] = serdeParams.separators[i]; + } + + for (byte asciival = 4; asciival <= MAX_CTRL_CHARS; asciival++) { + + //use only control chars that are very unlikely to be part of the string + // the following might/likely to be used in text files for strings + // 9 (horizontal tab, HT, \t, ^I) + // 10 (line feed, LF, \n, ^J), + // 12 (form feed, FF, \f, ^L), + // 13 (carriage return, CR, \r, ^M), + // 27 (escape, ESC, \e [GCC only], ^[). + + //reserving the following values for future dynamic level impl + // 30 + // 31 + + switch(asciival){ + case 9: + case 10: + case 12: + case 13: + case 27: + continue; + } + extendedSeparators[extendedSeparatorsIdx++] = asciival; + } + + serdeParams.separators = + Arrays.copyOfRange(extendedSeparators, 0, extendedSeparatorsIdx); } serdeParams.nullString = tbl.getProperty( @@ -272,6 +325,7 @@ public static SerDeParameters initSerdeParams(Configuration job, * @return The deserialized row Object. * @see SerDe#deserialize(Writable) */ + @Override public Object deserialize(Writable field) throws SerDeException { if (byteArrayRef == null) { byteArrayRef = new ByteArrayRef(); @@ -297,6 +351,7 @@ public Object deserialize(Writable field) throws SerDeException { /** * Returns the ObjectInspector for the row. */ + @Override public ObjectInspector getObjectInspector() throws SerDeException { return cachedObjectInspector; } @@ -306,6 +361,7 @@ public ObjectInspector getObjectInspector() throws SerDeException { * * @see SerDe#getSerializedClass() */ + @Override public Class getSerializedClass() { return Text.class; } @@ -324,6 +380,7 @@ public ObjectInspector getObjectInspector() throws SerDeException { * @throws IOException * @see SerDe#serialize(Object, ObjectInspector) */ + @Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { @@ -410,11 +467,12 @@ protected void serializeField(ByteStream.Output out, Object obj, ObjectInspector * 128. Negative byte values (or byte values >= 128) are never * escaped. * @throws IOException + * @throws SerDeException */ public static void serialize(ByteStream.Output out, Object obj, ObjectInspector objInspector, byte[] separators, int level, Text nullSequence, boolean escaped, byte escapeChar, boolean[] needsEscape) - throws IOException { + throws IOException, SerDeException { if (obj == null) { out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); @@ -430,7 +488,7 @@ public static void serialize(ByteStream.Output out, Object obj, needsEscape); return; case LIST: - separator = (char) separators[level]; + separator = (char) LazyUtils.getSeparator(separators, level); ListObjectInspector loi = (ListObjectInspector) objInspector; list = loi.getList(obj); ObjectInspector eoi = loi.getListElementObjectInspector(); @@ -447,8 +505,10 @@ public static void serialize(ByteStream.Output out, Object obj, } return; case MAP: - separator = (char) separators[level]; - char keyValueSeparator = (char) separators[level + 1]; + separator = (char) LazyUtils.getSeparator(separators, level); + char keyValueSeparator = + (char) LazyUtils.getSeparator(separators, level + 1); + MapObjectInspector moi = (MapObjectInspector) objInspector; ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); @@ -472,7 +532,7 @@ public static void serialize(ByteStream.Output out, Object obj, } return; case STRUCT: - separator = (char) separators[level]; + separator = (char) LazyUtils.getSeparator(separators, level); StructObjectInspector soi = (StructObjectInspector) objInspector; List fields = soi.getAllStructFieldRefs(); list = soi.getStructFieldsDataAsList(obj); @@ -490,7 +550,7 @@ public static void serialize(ByteStream.Output out, Object obj, } return; case UNION: - separator = (char) separators[level]; + separator = (char) LazyUtils.getSeparator(separators, level); UnionObjectInspector uoi = (UnionObjectInspector) objInspector; List ois = uoi.getObjectInspectors(); if (ois == null) { @@ -517,6 +577,7 @@ public static void serialize(ByteStream.Output out, Object obj, * Returns the statistics after (de)serialization) */ + @Override public SerDeStats getSerDeStats() { // must be different assert (lastOperationSerialize != lastOperationDeserialize); diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java index 27ed4ef..31cc270 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java @@ -367,6 +367,30 @@ public static void extractColumnInfo(Properties tbl, SerDeParameters serdeParams return Arrays.copyOf(sourceBw.getBytes(), sourceBw.getLength()); } + /** + * Utility function to get separator for current level used in serialization. + * Used to get a better log message when out of bound lookup happens + * @param separators - array of separators byte, byte at index x indicates + * separator used at that level + * @param level - nesting level + * @return separator at given level + * @throws SerDeException + */ + static byte getSeparator(byte[] separators, int level) throws SerDeException { + try{ + return separators[level]; + }catch(ArrayIndexOutOfBoundsException e){ + String msg = "Number of levels of nesting supported for " + + "LazySimpleSerde is " + (separators.length - 1) + + " Unable to work with level " + level; + if(separators.length < 9){ + msg += ". Use " + LazySimpleSerDe.SERIALIZATION_EXTEND_NESTING_LEVELS + + " serde property for tables using LazySimpleSerde."; + } + throw new SerDeException(msg, e); + } + } + private LazyUtils() { // prevent instantiation } diff --git serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java index 99628dc..24210f2 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java +++ serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java @@ -19,13 +19,21 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; import junit.framework.TestCase; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.io.IntWritable; @@ -37,6 +45,10 @@ */ public class TestLazyArrayMapStruct extends TestCase { + // nesting level limits + static final int EXTENDED_LEVEL_THRESHOLD = 24; + static final int DEFAULT_LEVEL_THRESHOLD = 8; + /** * Test the LazyArray class. */ @@ -325,4 +337,287 @@ public void testLazyUnion() throws Throwable { throw e; } } + + /** + * Test the LazyArray class with multiple levels of nesting + */ + public void testLazyArrayNested() throws Throwable { + for(int i = 2; i < EXTENDED_LEVEL_THRESHOLD; i++ ){ + testNestedinArrayAtLevelExtended(i, ObjectInspector.Category.LIST); + } + } + + /** + * Test the LazyArray class with multiple levels of nesting + */ + public void testLazyArrayNestedExceedLimit() throws Throwable { + checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD, ObjectInspector.Category.LIST); + } + + private void checkExtendedLimitExceeded(int maxLevel, Category type) { + boolean foundException = false; + try { + testNestedinArrayAtLevelExtended(maxLevel, type); + }catch (SerDeException serdeEx){ + foundException = true; + } + assertTrue("Got exception for exceeding nesting limit" , foundException); + } + + /** + * Test the LazyArray class with multiple levels of nesting, when nesting + * levels are not extended + */ + public void testLazyArrayNestedExceedLimitNotExtended() throws Throwable { + checkNotExtendedLimitExceeded(DEFAULT_LEVEL_THRESHOLD, + ObjectInspector.Category.LIST); + } + + /** + * Test the LazyMap class with multiple levels of nesting, when nesting + * levels are not extended + */ + public void testLazyMapNestedExceedLimitNotExtended() throws Throwable { + checkNotExtendedLimitExceeded(DEFAULT_LEVEL_THRESHOLD-1, + ObjectInspector.Category.MAP); + } + + /** + * Test the LazyMap class with multiple levels of nesting, when nesting + * levels are not extended + */ + public void testLazyStructNestedExceedLimitNotExtended() throws Throwable { + checkNotExtendedLimitExceeded(DEFAULT_LEVEL_THRESHOLD, + ObjectInspector.Category.STRUCT); + } + + /** + * Test the LazyMap class with multiple levels of nesting, when nesting + * levels are not extended + */ + public void testLazyUnionNestedExceedLimitNotExtended() throws Throwable { + checkNotExtendedLimitExceeded(DEFAULT_LEVEL_THRESHOLD, + ObjectInspector.Category.UNION); + } + + private void checkNotExtendedLimitExceeded(int maxLevel, Category type) { + boolean foundException = false; + try { + testNestedinArrayAtLevel(maxLevel, type, new Properties()); + }catch (SerDeException serdeEx){ + foundException = true; + } + assertTrue("Expected exception for exceeding nesting limit" , foundException); + } + + /** + * Test the LazyMap class with multiple levels of nesting + */ + public void testLazyMapNested() throws Throwable { + //map max nesting level is one less because it uses an additional separator + for(int i = 2; i < EXTENDED_LEVEL_THRESHOLD - 1; i++ ){ + testNestedinArrayAtLevelExtended(i, ObjectInspector.Category.MAP); + } + } + + /** + * Test the LazyMap class with multiple levels of nesting + */ + public void testLazyMapNestedExceedLimit() throws Throwable { + //map max nesting level is one less because it uses an additional separator + checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD - 1, ObjectInspector.Category.MAP); + } + + /** + * Test the LazyUnion class with multiple levels of nesting + */ + public void testLazyUnionNested() throws Throwable { + for(int i = 2; i < EXTENDED_LEVEL_THRESHOLD; i++ ){ + testNestedinArrayAtLevelExtended(i, ObjectInspector.Category.UNION); + } + } + + /** + * Test the LazyUnion class with multiple levels of nesting + */ + public void testLazyUnionNestedExceedLimit() throws Throwable { + checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD, ObjectInspector.Category.UNION); + } + + /** + * Test the LazyStruct class with multiple levels of nesting + */ + public void testLazyStructNested() throws Throwable { + for(int i = 2; i < EXTENDED_LEVEL_THRESHOLD; i++ ){ + testNestedinArrayAtLevelExtended(i, ObjectInspector.Category.STRUCT); + } + } + + /** + * Verify the serialized format for given type dtype, when it is nested in an + * array with nestingLevel levels. with extended nesting enabled. + * @param nestingLevel + * @param dtype + * @throws SerDeException + */ + private void testNestedinArrayAtLevelExtended(int nestingLevel, + ObjectInspector.Category dtype) throws SerDeException { + Properties tableProp = new Properties(); + tableProp.setProperty(LazySimpleSerDe.SERIALIZATION_EXTEND_NESTING_LEVELS, "true"); + testNestedinArrayAtLevel(nestingLevel, dtype, tableProp); + } + + /** + * Test the LazyStruct class with multiple levels of nesting + */ + public void testLazyStructNestedExceedLimit() throws Throwable { + checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD, ObjectInspector.Category.STRUCT); + } + + /** + * @param nestingLevel + * @param dtype + * @param tableProp + * @throws SerDeException + */ + private void testNestedinArrayAtLevel(int nestingLevel, + ObjectInspector.Category dtype, Properties tableProp) throws SerDeException { + + //create type with nestingLevel levels of nesting + //set inner schema for dtype + String inSchema = null; + switch(dtype){ + case LIST: + inSchema = "array"; + break; + case MAP: + inSchema = "map"; + break; + case STRUCT: + inSchema = "struct"; + break; + case UNION: + inSchema = "uniontype"; + break; + default : + fail("type not supported by test case"); + } + + StringBuilder schema = new StringBuilder(inSchema); + for(int i = 0; i < nestingLevel - 1; i++){ + schema.insert(0, "array<"); + schema.append(">"); + } + System.err.println("Testing nesting level " + nestingLevel + + ". Using schema " + schema); + + + // Create the SerDe + LazySimpleSerDe serDe = new LazySimpleSerDe(); + Configuration conf = new Configuration(); + tableProp.setProperty("columns", "narray"); + tableProp.setProperty("columns.types", schema.toString()); + serDe.initialize(conf, tableProp); + + //create the serialized string for type + byte[] separators = serDe.serdeParams.getSeparators(); + System.err.println("Using separator " + (char)separators[nestingLevel]); + byte [] serializedRow = null; + switch(dtype){ + case LIST: + serializedRow = new byte[] {'8',separators[nestingLevel],'9'}; + break; + case MAP: + byte kvSep = separators[nestingLevel+1]; + byte kvPairSep = separators[nestingLevel]; + serializedRow = new byte[] {'1', kvSep, '1', kvPairSep, '2', kvSep, '2'}; + break; + case STRUCT: + serializedRow = new byte[] {'8',separators[nestingLevel],'9'}; + break; + case UNION: + serializedRow = new byte[] {'0',separators[nestingLevel],'9'}; + break; + default : + fail("type not supported by test case"); + } + + + //create LazyStruct with serialized string with expected separators + StructObjectInspector oi = (StructObjectInspector) serDe + .getObjectInspector(); + LazyStruct struct = (LazyStruct) LazyFactory.createLazyObject(oi); + + TestLazyPrimitive.initLazyObject(struct, serializedRow, 0, serializedRow.length); + + + //Get fields out of the lazy struct and check if they match expected + // results + //Get first level array + LazyArray array = (LazyArray) struct.getField(0); + + //Peel off the n-1 levels to get to the underlying array + for(int i = 0; i < nestingLevel - 2; i++){ + array = (LazyArray) array.getListElementObject(0); + } + + //verify the serialized format for dtype + switch(dtype){ + case LIST: + LazyArray array1 = (LazyArray) array.getListElementObject(0); + //check elements of the innermost array + assertEquals(2, array1.getListLength()); + assertEquals(new ByteWritable((byte) 8), ((LazyByte) array1 + .getListElementObject(0)).getWritableObject()); + assertEquals(new ByteWritable((byte) 9), ((LazyByte) array1 + .getListElementObject(1)).getWritableObject()); + break; + + case MAP: + LazyMap lazyMap = (LazyMap) array.getListElementObject(0); + Map map = lazyMap.getMap(); + System.err.println(map); + assertEquals(2, map.size()); + Iterator> it = map.entrySet().iterator(); + + Entry e1 = it.next(); + assertEquals(e1.getKey().getWritableObject(), new Text(new byte[]{'1'}) ); + assertEquals(e1.getValue().getWritableObject(), new IntWritable(1) ); + + Entry e2 = it.next(); + assertEquals(e2.getKey().getWritableObject(), new Text(new byte[]{'2'}) ); + assertEquals(e2.getValue().getWritableObject(), new IntWritable(2) ); + break; + + case STRUCT: + LazyStruct innerStruct = (LazyStruct) array.getListElementObject(0); + //check elements of the innermost struct + assertEquals(2, innerStruct.getFieldsAsList().size()); + assertEquals(new Text(new byte[]{'8'}), + ((LazyString) innerStruct.getField(0)).getWritableObject()); + assertEquals(new ByteWritable((byte) 9), + ((LazyByte) innerStruct.getField(1)).getWritableObject()); + break; + + case UNION: + LazyUnion lazyUnion = (LazyUnion) array.getListElementObject(0); + //check elements of the innermost union + assertEquals(new Text(new byte[]{'9'}), + ((LazyString)lazyUnion.getField()).getWritableObject()); + break; + + + default : + fail("type not supported by test case"); + } + + //test serialization + Text serializedText = + (Text) serDe.serialize(struct.getObject(), serDe.getObjectInspector()); + org.junit.Assert.assertArrayEquals(serializedRow, serializedText.getBytes()); + + } + + + }