diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java index a124938..5870a50 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java @@ -217,7 +217,7 @@ private static StructTypeInfo pruneFromPaths( private static void pruneFromSinglePath(PrunedStructTypeInfo prunedInfo, String path) { Preconditions.checkArgument(prunedInfo != null, - "PrunedStructTypeInfo for path " + path + " should not be null"); + "PrunedStructTypeInfo for path '" + path + "' should not be null"); int index = path.indexOf('.'); if (index < 0) { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java index e9af7a7..2a0c469 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java @@ -39,8 +39,10 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.SelectDesc; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIndex; import static org.apache.hadoop.hive.ql.optimizer.FieldNode.mergeFieldNodes; @@ -242,10 +244,24 @@ private static void getNestedColsFromExprNodeDesc( p.addFieldNodes(pathToRoot); paths.add(p); } else if (desc instanceof ExprNodeFieldDesc) { - String f = ((ExprNodeFieldDesc) desc).getFieldName(); + ExprNodeFieldDesc fieldDesc = (ExprNodeFieldDesc) desc; + ExprNodeDesc childDesc = fieldDesc.getDesc(); + + // Check cases for arr[i].f and map[key].v + // For these we should not generate paths like arr.f or map.v + // Otherwise we would have a mismatch between type info and path + if (childDesc instanceof ExprNodeGenericFuncDesc) { + ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) childDesc; + if (funcDesc.getGenericUDF() instanceof GenericUDFIndex) { + getNestedColsFromExprNodeDesc(funcDesc, pathToRoot, paths); + return; + } + } + + String f = fieldDesc.getFieldName(); FieldNode p = new FieldNode(f); p.addFieldNodes(pathToRoot); - getNestedColsFromExprNodeDesc(((ExprNodeFieldDesc) desc).getDesc(), p, paths); + getNestedColsFromExprNodeDesc(childDesc, p, paths); } else { List children = desc.getChildren(); if (children == null || children.isEmpty()) { diff --git ql/src/test/queries/clientpositive/nested_column_pruning.q ql/src/test/queries/clientpositive/nested_column_pruning.q index 28b974e..ac261d1 100644 --- ql/src/test/queries/clientpositive/nested_column_pruning.q +++ ql/src/test/queries/clientpositive/nested_column_pruning.q @@ -10,13 +10,15 @@ CREATE TABLE nested_tbl_1 ( a int, s1 struct, f6: int>, s2 struct, f11: map>>, - s3 struct>> + s3 struct>>, + s4 map> ) STORED AS PARQUET; INSERT INTO TABLE nested_tbl_1 SELECT 1, named_struct('f1', false, 'f2', 'foo', 'f3', named_struct('f4', 4, 'f5', cast(5.0 as double)), 'f6', 4), named_struct('f7', 'f7', 'f8', named_struct('f9', true, 'f10', array(10, 11), 'f11', map('key1', true, 'key2', false))), - named_struct('f12', array(named_struct('f13', 'foo', 'f14', 14), named_struct('f13', 'bar', 'f14', 28))) + named_struct('f12', array(named_struct('f13', 'foo', 'f14', 14), named_struct('f13', 'bar', 'f14', 28))), + map('key1', named_struct('f15', 1), 'key2', named_struct('f15', 2)) FROM dummy; DROP TABLE IF EXISTS nested_tbl_2; @@ -25,7 +27,8 @@ CREATE TABLE nested_tbl_2 LIKE nested_tbl_1; INSERT INTO TABLE nested_tbl_2 SELECT 2, named_struct('f1', true, 'f2', 'bar', 'f3', named_struct('f4', 4, 'f5', cast(6.5 as double)), 'f6', 4), named_struct('f7', 'f72', 'f8', named_struct('f9', false, 'f10', array(20, 22), 'f11', map('key3', true, 'key4', false))), - named_struct('f12', array(named_struct('f13', 'bar', 'f14', 28), named_struct('f13', 'foo', 'f14', 56))) + named_struct('f12', array(named_struct('f13', 'bar', 'f14', 28), named_struct('f13', 'foo', 'f14', 56))), + map('key3', named_struct('f15', 3), 'key4', named_struct('f15', 4)) FROM dummy; -- Testing only select statements @@ -110,3 +113,24 @@ SELECT t1.s1.f3.f5, t2.s2.f8 FROM nested_tbl_1 t1 JOIN nested_tbl_1 t2 ON t1.s1.f3.f4 = t2.s1.f6 WHERE t2.s2.f8.f9 == TRUE; + + +-- Testing select struct field from elements in array or map + +EXPLAIN +SELECT count(s1.f6), s3.f12[0].f14 +FROM nested_tbl_1 +GROUP BY s3.f12[0].f14; + +SELECT count(s1.f6), s3.f12[0].f14 +FROM nested_tbl_1 +GROUP BY s3.f12[0].f14; + +EXPLAIN +SELECT count(s1.f6), s4['key1'].f15 +FROM nested_tbl_1 +GROUP BY s4['key1'].f15; + +SELECT count(s1.f6), s4['key1'].f15 +FROM nested_tbl_1 +GROUP BY s4['key1'].f15; diff --git ql/src/test/results/clientpositive/nested_column_pruning.q.out ql/src/test/results/clientpositive/nested_column_pruning.q.out index f01e3ea..2480d4e 100644 --- ql/src/test/results/clientpositive/nested_column_pruning.q.out +++ ql/src/test/results/clientpositive/nested_column_pruning.q.out @@ -27,7 +27,8 @@ PREHOOK: query: CREATE TABLE nested_tbl_1 ( a int, s1 struct, f6: int>, s2 struct, f11: map>>, - s3 struct>> + s3 struct>>, + s4 map> ) STORED AS PARQUET PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -36,7 +37,8 @@ POSTHOOK: query: CREATE TABLE nested_tbl_1 ( a int, s1 struct, f6: int>, s2 struct, f11: map>>, - s3 struct>> + s3 struct>>, + s4 map> ) STORED AS PARQUET POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -44,7 +46,8 @@ POSTHOOK: Output: default@nested_tbl_1 PREHOOK: query: INSERT INTO TABLE nested_tbl_1 SELECT 1, named_struct('f1', false, 'f2', 'foo', 'f3', named_struct('f4', 4, 'f5', cast(5.0 as double)), 'f6', 4), named_struct('f7', 'f7', 'f8', named_struct('f9', true, 'f10', array(10, 11), 'f11', map('key1', true, 'key2', false))), - named_struct('f12', array(named_struct('f13', 'foo', 'f14', 14), named_struct('f13', 'bar', 'f14', 28))) + named_struct('f12', array(named_struct('f13', 'foo', 'f14', 14), named_struct('f13', 'bar', 'f14', 28))), + map('key1', named_struct('f15', 1), 'key2', named_struct('f15', 2)) FROM dummy PREHOOK: type: QUERY PREHOOK: Input: default@dummy @@ -52,7 +55,8 @@ PREHOOK: Output: default@nested_tbl_1 POSTHOOK: query: INSERT INTO TABLE nested_tbl_1 SELECT 1, named_struct('f1', false, 'f2', 'foo', 'f3', named_struct('f4', 4, 'f5', cast(5.0 as double)), 'f6', 4), named_struct('f7', 'f7', 'f8', named_struct('f9', true, 'f10', array(10, 11), 'f11', map('key1', true, 'key2', false))), - named_struct('f12', array(named_struct('f13', 'foo', 'f14', 14), named_struct('f13', 'bar', 'f14', 28))) + named_struct('f12', array(named_struct('f13', 'foo', 'f14', 14), named_struct('f13', 'bar', 'f14', 28))), + map('key1', named_struct('f15', 1), 'key2', named_struct('f15', 2)) FROM dummy POSTHOOK: type: QUERY POSTHOOK: Input: default@dummy @@ -61,6 +65,7 @@ POSTHOOK: Lineage: nested_tbl_1.a SIMPLE [] POSTHOOK: Lineage: nested_tbl_1.s1 EXPRESSION [] POSTHOOK: Lineage: nested_tbl_1.s2 EXPRESSION [] POSTHOOK: Lineage: nested_tbl_1.s3 EXPRESSION [] +POSTHOOK: Lineage: nested_tbl_1.s4 EXPRESSION [] PREHOOK: query: DROP TABLE IF EXISTS nested_tbl_2 PREHOOK: type: DROPTABLE POSTHOOK: query: DROP TABLE IF EXISTS nested_tbl_2 @@ -76,7 +81,8 @@ POSTHOOK: Output: default@nested_tbl_2 PREHOOK: query: INSERT INTO TABLE nested_tbl_2 SELECT 2, named_struct('f1', true, 'f2', 'bar', 'f3', named_struct('f4', 4, 'f5', cast(6.5 as double)), 'f6', 4), named_struct('f7', 'f72', 'f8', named_struct('f9', false, 'f10', array(20, 22), 'f11', map('key3', true, 'key4', false))), - named_struct('f12', array(named_struct('f13', 'bar', 'f14', 28), named_struct('f13', 'foo', 'f14', 56))) + named_struct('f12', array(named_struct('f13', 'bar', 'f14', 28), named_struct('f13', 'foo', 'f14', 56))), + map('key3', named_struct('f15', 3), 'key4', named_struct('f15', 4)) FROM dummy PREHOOK: type: QUERY PREHOOK: Input: default@dummy @@ -84,7 +90,8 @@ PREHOOK: Output: default@nested_tbl_2 POSTHOOK: query: INSERT INTO TABLE nested_tbl_2 SELECT 2, named_struct('f1', true, 'f2', 'bar', 'f3', named_struct('f4', 4, 'f5', cast(6.5 as double)), 'f6', 4), named_struct('f7', 'f72', 'f8', named_struct('f9', false, 'f10', array(20, 22), 'f11', map('key3', true, 'key4', false))), - named_struct('f12', array(named_struct('f13', 'bar', 'f14', 28), named_struct('f13', 'foo', 'f14', 56))) + named_struct('f12', array(named_struct('f13', 'bar', 'f14', 28), named_struct('f13', 'foo', 'f14', 56))), + map('key3', named_struct('f15', 3), 'key4', named_struct('f15', 4)) FROM dummy POSTHOOK: type: QUERY POSTHOOK: Input: default@dummy @@ -93,6 +100,7 @@ POSTHOOK: Lineage: nested_tbl_2.a SIMPLE [] POSTHOOK: Lineage: nested_tbl_2.s1 EXPRESSION [] POSTHOOK: Lineage: nested_tbl_2.s2 EXPRESSION [] POSTHOOK: Lineage: nested_tbl_2.s3 EXPRESSION [] +POSTHOOK: Lineage: nested_tbl_2.s4 EXPRESSION [] PREHOOK: query: -- Testing only select statements EXPLAIN SELECT a FROM nested_tbl_1 @@ -111,14 +119,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: nested_tbl_1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -153,14 +161,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: nested_tbl_1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s1.f1 (type: boolean) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -195,14 +203,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: nested_tbl_1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s1.f1 (type: boolean), s1.f2 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -239,14 +247,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: nested_tbl_1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s1.f3 (type: struct), s1.f3.f4 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -283,14 +291,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: nested_tbl_1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s1.f3.f5 (type: double) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -327,14 +335,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: nested_tbl_1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s1.f3.f4 (type: int), s2.f8.f9 (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -373,17 +381,17 @@ STAGE PLANS: Map Operator Tree: TableScan alias: nested_tbl_1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (s1.f1 = false) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s1.f2 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -418,17 +426,17 @@ STAGE PLANS: Map Operator Tree: TableScan alias: nested_tbl_1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (s1.f3.f4 = 4) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s1.f3.f5 (type: double) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -463,17 +471,17 @@ STAGE PLANS: Map Operator Tree: TableScan alias: nested_tbl_1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((s1.f2 = 'foo') and (size(s2.f8.f10) > 1) and s2.f8.f11['key1']) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s2.f8 (type: struct,f11:map>) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -516,32 +524,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: nested_tbl_1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Lateral View Forward - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s3 (type: struct>>) outputColumnNames: s3 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator - outputColumnNames: _col3, _col7 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col3, _col8 + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE Lateral View Forward - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col7 (type: int) - outputColumnNames: _col7 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + expressions: _col8 (type: int) + outputColumnNames: _col8 + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator - outputColumnNames: _col7, _col8 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col8, _col9 + Statistics: Num rows: 4 Data size: 20 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col7 (type: int), _col8 (type: struct) + expressions: _col8 (type: int), _col9 (type: struct) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 20 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 20 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -549,20 +557,20 @@ STAGE PLANS: Select Operator expressions: _col3.f12 (type: array>) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE UDTF Operator - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE function name: explode Lateral View Join Operator - outputColumnNames: _col7, _col8 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col8, _col9 + Statistics: Num rows: 4 Data size: 20 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col7 (type: int), _col8 (type: struct) + expressions: _col8 (type: int), _col9 (type: struct) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 20 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 20 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -570,29 +578,29 @@ STAGE PLANS: Select Operator expressions: s2.f8.f10 (type: array) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE UDTF Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE function name: explode Lateral View Join Operator - outputColumnNames: _col3, _col7 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col3, _col8 + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE Lateral View Forward - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col7 (type: int) - outputColumnNames: _col7 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + expressions: _col8 (type: int) + outputColumnNames: _col8 + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator - outputColumnNames: _col7, _col8 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col8, _col9 + Statistics: Num rows: 4 Data size: 20 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col7 (type: int), _col8 (type: struct) + expressions: _col8 (type: int), _col9 (type: struct) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 20 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 20 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -600,20 +608,20 @@ STAGE PLANS: Select Operator expressions: _col3.f12 (type: array>) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE UDTF Operator - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE function name: explode Lateral View Join Operator - outputColumnNames: _col7, _col8 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col8, _col9 + Statistics: Num rows: 4 Data size: 20 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col7 (type: int), _col8 (type: struct) + expressions: _col8 (type: int), _col9 (type: struct) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 20 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 20 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -657,14 +665,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: nested_tbl_1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (s2.f8.f10[1] pmod s1.f3.f4) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -703,22 +711,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: nested_tbl_1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s1.f3.f5 (type: double), s1.f3.f4 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col1) keys: _col0 (type: double) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: double) sort order: + Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -726,10 +734,10 @@ STAGE PLANS: keys: KEY._col0 (type: double) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -764,22 +772,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: nested_tbl_1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s1.f3 (type: struct), s1.f3.f4 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col1) keys: _col0 (type: struct) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: struct) sort order: + Map-reduce partition columns: _col0 (type: struct) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -787,10 +795,10 @@ STAGE PLANS: keys: KEY._col0 (type: struct) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -826,22 +834,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: nested_tbl_1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s1.f3 (type: struct), s1.f3.f4 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col1) keys: _col0 (type: struct) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: struct) sort order: + Map-reduce partition columns: _col0 (type: struct) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -849,7 +857,7 @@ STAGE PLANS: keys: KEY._col0 (type: struct) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -864,16 +872,16 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: struct) sort order: + - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -918,32 +926,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s1 (type: struct,f6:int>) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0.f3.f4 (type: int) sort order: + Map-reduce partition columns: _col0.f3.f4 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: struct,f6:int>) TableScan alias: t2 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (s2.f8.f9 = false) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s1 (type: struct,f6:int>), s2 (type: struct,f11:map>>) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0.f6 (type: int) sort order: + Map-reduce partition columns: _col0.f6 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: struct,f11:map>>) Reduce Operator Tree: Join Operator @@ -953,14 +961,14 @@ STAGE PLANS: 0 _col0.f3.f4 (type: int) 1 _col0.f6 (type: int) outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0.f3.f5 (type: double), _col2.f8 (type: struct,f11:map>) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1009,32 +1017,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s1 (type: struct,f6:int>) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0.f3.f4 (type: int) sort order: + Map-reduce partition columns: _col0.f3.f4 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: struct,f6:int>) TableScan alias: t2 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (s2.f8.f9 = true) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s1 (type: struct,f6:int>), s2 (type: struct,f11:map>>) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0.f6 (type: int) sort order: + Map-reduce partition columns: _col0.f6 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: struct,f11:map>>) Reduce Operator Tree: Join Operator @@ -1044,14 +1052,14 @@ STAGE PLANS: 0 _col0.f3.f4 (type: int) 1 _col0.f6 (type: int) outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0.f3.f5 (type: double), _col2.f8 (type: struct,f11:map>) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1078,3 +1086,157 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@nested_tbl_1 #### A masked pattern was here #### 5.0 {"f9":true,"f10":[10,11],"f11":{"key1":true,"key2":false}} +PREHOOK: query: -- Testing select struct field from elements in array or map + +EXPLAIN +SELECT count(s1.f6), s3.f12[0].f14 +FROM nested_tbl_1 +GROUP BY s3.f12[0].f14 +PREHOOK: type: QUERY +POSTHOOK: query: -- Testing select struct field from elements in array or map + +EXPLAIN +SELECT count(s1.f6), s3.f12[0].f14 +FROM nested_tbl_1 +GROUP BY s3.f12[0].f14 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: nested_tbl_1 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: s3.f12[0].f14 (type: int), s1.f6 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col1) + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: bigint), _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT count(s1.f6), s3.f12[0].f14 +FROM nested_tbl_1 +GROUP BY s3.f12[0].f14 +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(s1.f6), s3.f12[0].f14 +FROM nested_tbl_1 +GROUP BY s3.f12[0].f14 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +1 14 +PREHOOK: query: EXPLAIN +SELECT count(s1.f6), s4['key1'].f15 +FROM nested_tbl_1 +GROUP BY s4['key1'].f15 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT count(s1.f6), s4['key1'].f15 +FROM nested_tbl_1 +GROUP BY s4['key1'].f15 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: nested_tbl_1 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: s4['key1'].f15 (type: int), s1.f6 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col1) + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: bigint), _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT count(s1.f6), s4['key1'].f15 +FROM nested_tbl_1 +GROUP BY s4['key1'].f15 +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(s1.f6), s4['key1'].f15 +FROM nested_tbl_1 +GROUP BY s4['key1'].f15 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +1 1