diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java index f5c42ef..bba06b6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java @@ -78,8 +78,8 @@ public class LateralViewJoinOperator extends Operator { // The expected tags from the parent operators. See processOp() before // changing the tags. - static final int SELECT_TAG = 0; - static final int UDTF_TAG = 1; + public static final int SELECT_TAG = 0; + public static final int UDTF_TAG = 1; @Override protected void initializeOp(Configuration hconf) throws HiveException { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java index 18933fc..55ac3e0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java @@ -94,6 +94,8 @@ public class ColumnPruner implements Transform { .getTableScanProc()); opRules.put(new RuleRegExp("R8", "LVJ%"), ColumnPrunerProcFactory .getLateralViewJoinProc()); + opRules.put(new RuleRegExp("R9", "LVF%"), ColumnPrunerProcFactory + .getLateralViewForwardProc()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(ColumnPrunerProcFactory diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java index 01c6afa..74599b7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.parse.OpParseContext; +import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.SelectDesc; @@ -155,4 +156,18 @@ public class ColumnPrunerProcCtx implements NodeProcessorCtx { return cols; } + + /** + * Create the list of internal columns for select tag of LV + */ + public List getSelectColsFromLVJoin(RowResolver rr, + List colList) throws SemanticException { + List columns = new ArrayList(); + for (String col : colList) { + if (rr.reverseLookup(col) != null) { + columns.add(col); + } + } + return columns; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java index 2b40eb9..d1e8cc2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.FilterOperator; import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; +import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator; import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator; import org.apache.hadoop.hive.ql.exec.LimitOperator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; @@ -56,6 +57,7 @@ import org.apache.hadoop.hive.ql.parse.OpParseContext; import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.AggregationDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.JoinDesc; @@ -314,9 +316,8 @@ public final class ColumnPrunerProcFactory { Object... nodeOutputs) throws SemanticException { LateralViewJoinOperator op = (LateralViewJoinOperator) nd; ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx; - List cols = new ArrayList(); + List cols = cppCtx.genColLists(op); - cols = cppCtx.genColLists(op); Map colExprMap = op.getColumnExprMap(); // As columns go down the DAG, the LVJ will transform internal column @@ -332,6 +333,22 @@ public final class ColumnPrunerProcFactory { colsAfterReplacement.add(col); } } + // this is SEL(*) cols + UDTF cols + List outputCols = op.getConf().getOutputInternalColNames(); + if (outputCols.size() != cols.size()) { + // cause we cannot prune columns from UDTF branch currently, extract + // columns from SEL(*) branch only and append all columns from UDTF branch to it + ArrayList newColNames = new ArrayList(); + for (String col : cols) { + int index = outputCols.indexOf(col); + // colExprMap.size() == size of cols from SEL(*) branch + if (index >= 0 && index < colExprMap.size()) { + newColNames.add(col); + } + } + newColNames.addAll(outputCols.subList(colExprMap.size(), outputCols.size())); + op.getConf().setOutputInternalColNames(newColNames); + } cppCtx.getPrunedColLists().put(op, colsAfterReplacement); @@ -340,6 +357,44 @@ public final class ColumnPrunerProcFactory { } /** + * The Node Processor for Column Pruning on Lateral View Forward Operators. + */ + public static class ColumnPrunerLateralViewForwardProc extends ColumnPrunerDefaultProc { + @Override + public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, + Object... nodeOutputs) throws SemanticException { + super.process(nd, stack, ctx, nodeOutputs); + LateralViewForwardOperator op = (LateralViewForwardOperator) nd; + ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx; + + // get the SEL(*) branch + Operator select = op.getChildOperators().get(LateralViewJoinOperator.SELECT_TAG); + + // these are from ColumnPrunerSelectProc + List cols = cppCtx.getPrunedColList(select); + RowResolver rr = cppCtx.getOpToParseCtxMap().get(op).getRowResolver(); + if (rr.getColumnInfos().size() != cols.size()) { + ArrayList colList = new ArrayList(); + ArrayList outputColNames = new ArrayList(); + for (String col : cols) { + // revert output cols of SEL(*) to ExprNodeColumnDesc + String[] tabcol = rr.reverseLookup(col); + ColumnInfo colInfo = rr.get(tabcol[0], tabcol[1]); + ExprNodeColumnDesc colExpr = new ExprNodeColumnDesc(colInfo.getType(), + colInfo.getInternalName(), colInfo.getTabAlias(), colInfo.getIsVirtualCol()); + colList.add(colExpr); + outputColNames.add(col); + } + // replace SEL(*) to SEL(exprs) + ((SelectDesc)select.getConf()).setSelStarNoCompute(false); + ((SelectDesc)select.getConf()).setColList(colList); + ((SelectDesc)select.getConf()).setOutputColumnNames(outputColNames); + } + return null; + } + } + + /** * The Node Processor for Column Pruning on Select Operators. */ public static class ColumnPrunerSelectProc implements NodeProcessor { @@ -347,8 +402,8 @@ public final class ColumnPrunerProcFactory { Object... nodeOutputs) throws SemanticException { SelectOperator op = (SelectOperator) nd; ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx; - List cols = new ArrayList(); + LateralViewJoinOperator lvJoin = null; if (op.getChildOperators() != null) { for (Operator child : op.getChildOperators()) { // If one of my children is a FileSink or Script, return all columns. @@ -364,11 +419,22 @@ public final class ColumnPrunerProcFactory { .put(op, cppCtx.getColsFromSelectExpr(op)); return null; } + if (op.getConf().isSelStarNoCompute() && child instanceof LateralViewJoinOperator) { + // this SEL is SEL(*) for LV + lvJoin = (LateralViewJoinOperator) child; + } } } - cols = cppCtx.genColLists(op); + List cols = cppCtx.genColLists(op); SelectDesc conf = op.getConf(); + + if (lvJoin != null) { + // get columns for SEL(*) from LVJ + RowResolver rr = cppCtx.getOpToParseCtxMap().get(op).getRowResolver(); + cppCtx.getPrunedColLists().put(op, cppCtx.getSelectColsFromLVJoin(rr, cols)); + return null; + } // The input to the select does not matter. Go over the expressions // and return the ones which have a marked column cppCtx.getPrunedColLists().put(op, @@ -541,6 +607,10 @@ public final class ColumnPrunerProcFactory { return new ColumnPrunerLateralViewJoinProc(); } + public static ColumnPrunerLateralViewForwardProc getLateralViewForwardProc() { + return new ColumnPrunerLateralViewForwardProc(); + } + /** * The Node Processor for Column Pruning on Join Operators. */ diff --git ql/src/test/queries/clientpositive/lateral_view_cp.q ql/src/test/queries/clientpositive/lateral_view_cp.q new file mode 100644 index 0000000..5e0b6fa --- /dev/null +++ ql/src/test/queries/clientpositive/lateral_view_cp.q @@ -0,0 +1,8 @@ +--HIVE 3226 +drop table array_valued_src; +create table array_valued_src (key string, value array); +insert overwrite table array_valued_src select key, array(value) from src; + +-- replace sel(*) to sel(exprs) for reflecting CP result properly +explain select count(val) from (select a.key as key, b.value as array_val from src a join array_valued_src b on a.key=b.key) i lateral view explode (array_val) c as val; +select count(val) from (select a.key as key, b.value as array_val from src a join array_valued_src b on a.key=b.key) i lateral view explode (array_val) c as val; diff --git ql/src/test/results/clientnegative/udf_assert_true.q.out ql/src/test/results/clientnegative/udf_assert_true.q.out index 93bbc84..8a63572 100644 --- ql/src/test/results/clientnegative/udf_assert_true.q.out +++ ql/src/test/results/clientnegative/udf_assert_true.q.out @@ -23,9 +23,8 @@ STAGE PLANS: alias: src Lateral View Forward Select Operator - SELECT * : (no compute) Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col2 Select Operator expressions: expr: assert_true((_col2 > 0)) @@ -46,7 +45,7 @@ STAGE PLANS: UDTF Operator function name: explode Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col2 Select Operator expressions: expr: assert_true((_col2 > 0)) @@ -95,9 +94,8 @@ STAGE PLANS: alias: src Lateral View Forward Select Operator - SELECT * : (no compute) Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col2 Select Operator expressions: expr: assert_true((_col2 < 2)) @@ -118,7 +116,7 @@ STAGE PLANS: UDTF Operator function name: explode Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col2 Select Operator expressions: expr: assert_true((_col2 < 2)) diff --git ql/src/test/results/clientnegative/udf_assert_true2.q.out ql/src/test/results/clientnegative/udf_assert_true2.q.out index 5cfd9e3..508a653 100644 --- ql/src/test/results/clientnegative/udf_assert_true2.q.out +++ ql/src/test/results/clientnegative/udf_assert_true2.q.out @@ -18,9 +18,8 @@ STAGE PLANS: alias: src Lateral View Forward Select Operator - SELECT * : (no compute) Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col2 Select Operator expressions: expr: (1 + assert_true((_col2 < 2))) @@ -41,7 +40,7 @@ STAGE PLANS: UDTF Operator function name: explode Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col2 Select Operator expressions: expr: (1 + assert_true((_col2 < 2))) diff --git ql/src/test/results/clientpositive/lateral_view.q.out ql/src/test/results/clientpositive/lateral_view.q.out index cfd2822..61bb6ea 100644 --- ql/src/test/results/clientpositive/lateral_view.q.out +++ ql/src/test/results/clientpositive/lateral_view.q.out @@ -159,9 +159,8 @@ STAGE PLANS: alias: src Lateral View Forward Select Operator - SELECT * : (no compute) Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col2 Select Operator expressions: expr: _col2 @@ -182,7 +181,7 @@ STAGE PLANS: UDTF Operator function name: explode Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col2 Select Operator expressions: expr: _col2 @@ -222,14 +221,16 @@ STAGE PLANS: alias: src Lateral View Forward Select Operator - SELECT * : (no compute) Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col2 Lateral View Forward Select Operator - SELECT * : (no compute) + expressions: + expr: _col2 + type: int + outputColumnNames: _col2 Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col2, _col3 Select Operator expressions: expr: _col2 @@ -252,7 +253,7 @@ STAGE PLANS: UDTF Operator function name: explode Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col2, _col3 Select Operator expressions: expr: _col2 @@ -275,12 +276,15 @@ STAGE PLANS: UDTF Operator function name: explode Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col2 Lateral View Forward Select Operator - SELECT * : (no compute) + expressions: + expr: _col2 + type: int + outputColumnNames: _col2 Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col2, _col3 Select Operator expressions: expr: _col2 @@ -303,7 +307,7 @@ STAGE PLANS: UDTF Operator function name: explode Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col2, _col3 Select Operator expressions: expr: _col2 @@ -345,14 +349,12 @@ STAGE PLANS: alias: src Lateral View Forward Select Operator - SELECT * : (no compute) Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col2 Lateral View Forward Select Operator - SELECT * : (no compute) Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col3 Select Operator expressions: expr: _col3 @@ -373,7 +375,7 @@ STAGE PLANS: UDTF Operator function name: explode Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col3 Select Operator expressions: expr: _col3 @@ -394,12 +396,11 @@ STAGE PLANS: UDTF Operator function name: explode Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col2 Lateral View Forward Select Operator - SELECT * : (no compute) Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col3 Select Operator expressions: expr: _col3 @@ -420,7 +421,7 @@ STAGE PLANS: UDTF Operator function name: explode Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col3 Select Operator expressions: expr: _col3 @@ -522,9 +523,8 @@ STAGE PLANS: alias: tmp_pyang_lv Lateral View Forward Select Operator - SELECT * : (no compute) Lateral View Join Operator - outputColumnNames: _col0, _col1 + outputColumnNames: _col1 Select Operator expressions: expr: _col1 @@ -545,7 +545,7 @@ STAGE PLANS: UDTF Operator function name: explode Lateral View Join Operator - outputColumnNames: _col0, _col1 + outputColumnNames: _col1 Select Operator expressions: expr: _col1 @@ -861,16 +861,17 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Select Operator expressions: - expr: _col1 - type: string expr: array(_col0) type: array - outputColumnNames: _col0, _col1 + outputColumnNames: _col1 Lateral View Forward Select Operator - SELECT * : (no compute) + expressions: + expr: _col1 + type: array + outputColumnNames: _col1 Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col1, _col2 Select Operator expressions: expr: _col1 @@ -892,7 +893,7 @@ STAGE PLANS: UDTF Operator function name: explode Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col1, _col2 Select Operator expressions: expr: _col1 diff --git ql/src/test/results/clientpositive/lateral_view_cp.q.out ql/src/test/results/clientpositive/lateral_view_cp.q.out new file mode 100644 index 0000000..7c342c1 --- /dev/null +++ ql/src/test/results/clientpositive/lateral_view_cp.q.out @@ -0,0 +1,177 @@ +PREHOOK: query: --HIVE 3226 +drop table array_valued_src +PREHOOK: type: DROPTABLE +POSTHOOK: query: --HIVE 3226 +drop table array_valued_src +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table array_valued_src (key string, value array) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table array_valued_src (key string, value array) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@array_valued_src +PREHOOK: query: insert overwrite table array_valued_src select key, array(value) from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@array_valued_src +POSTHOOK: query: insert overwrite table array_valued_src select key, array(value) from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@array_valued_src +POSTHOOK: Lineage: array_valued_src.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: array_valued_src.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- replace sel(*) to sel(exprs) for reflecting CP result properly +explain select count(val) from (select a.key as key, b.value as array_val from src a join array_valued_src b on a.key=b.key) i lateral view explode (array_val) c as val +PREHOOK: type: QUERY +POSTHOOK: query: -- replace sel(*) to sel(exprs) for reflecting CP result properly +explain select count(val) from (select a.key as key, b.value as array_val from src a join array_valued_src b on a.key=b.key) i lateral view explode (array_val) c as val +POSTHOOK: type: QUERY +POSTHOOK: Lineage: array_valued_src.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: array_valued_src.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_LATERAL_VIEW (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION explode (TOK_TABLE_OR_COL array_val)) val (TOK_TABALIAS c))) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src) a) (TOK_TABREF (TOK_TABNAME array_valued_src) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) value) array_val)))) i))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION count (TOK_TABLE_OR_COL val)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + i:a + TableScan + alias: a + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 0 + i:b + TableScan + alias: b + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: value + type: array + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {VALUE._col1} + handleSkewJoin: false + outputColumnNames: _col5 + Select Operator + expressions: + expr: _col5 + type: array + outputColumnNames: _col1 + Lateral View Forward + Select Operator + Lateral View Join Operator + outputColumnNames: _col2 + Select Operator + expressions: + expr: _col2 + type: string + outputColumnNames: _col2 + Group By Operator + aggregations: + expr: count(_col2) + bucketGroup: false + mode: hash + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + Select Operator + expressions: + expr: _col1 + type: array + outputColumnNames: _col0 + UDTF Operator + function name: explode + Lateral View Join Operator + outputColumnNames: _col2 + Select Operator + expressions: + expr: _col2 + type: string + outputColumnNames: _col2 + Group By Operator + aggregations: + expr: count(_col2) + bucketGroup: false + mode: hash + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: bigint + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(VALUE._col0) + bucketGroup: false + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: + expr: _col0 + type: bigint + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: select count(val) from (select a.key as key, b.value as array_val from src a join array_valued_src b on a.key=b.key) i lateral view explode (array_val) c as val +PREHOOK: type: QUERY +PREHOOK: Input: default@array_valued_src +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select count(val) from (select a.key as key, b.value as array_val from src a join array_valued_src b on a.key=b.key) i lateral view explode (array_val) c as val +POSTHOOK: type: QUERY +POSTHOOK: Input: default@array_valued_src +POSTHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: Lineage: array_valued_src.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: array_valued_src.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +1028 diff --git ql/src/test/results/clientpositive/lateral_view_ppd.q.out ql/src/test/results/clientpositive/lateral_view_ppd.q.out index 8b6920a..5463364 100644 --- ql/src/test/results/clientpositive/lateral_view_ppd.q.out +++ ql/src/test/results/clientpositive/lateral_view_ppd.q.out @@ -226,9 +226,16 @@ STAGE PLANS: alias: srcpart Lateral View Forward Select Operator - SELECT * : (no compute) + expressions: + expr: value + type: string + expr: ds + type: string + expr: hr + type: string + outputColumnNames: value, ds, hr Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3, _col4 + outputColumnNames: _col1, _col2, _col3, _col4 Select Operator expressions: expr: _col1 @@ -262,7 +269,7 @@ STAGE PLANS: UDTF Operator function name: explode Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3, _col4 + outputColumnNames: _col1, _col2, _col3, _col4 Select Operator expressions: expr: _col1 diff --git ql/src/test/results/clientpositive/udtf_json_tuple.q.out ql/src/test/results/clientpositive/udtf_json_tuple.q.out index a8aa05b..6611b16 100644 --- ql/src/test/results/clientpositive/udtf_json_tuple.q.out +++ ql/src/test/results/clientpositive/udtf_json_tuple.q.out @@ -63,9 +63,12 @@ STAGE PLANS: alias: a Lateral View Forward Select Operator - SELECT * : (no compute) + expressions: + expr: key + type: string + outputColumnNames: key Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6 Select Operator expressions: expr: _col0 @@ -118,7 +121,7 @@ STAGE PLANS: UDTF Operator function name: json_tuple Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6 Select Operator expressions: expr: _col0 @@ -297,9 +300,12 @@ STAGE PLANS: alias: a Lateral View Forward Select Operator - SELECT * : (no compute) + expressions: + expr: key + type: string + outputColumnNames: key Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6 Select Operator expressions: expr: _col0 @@ -340,7 +346,7 @@ STAGE PLANS: UDTF Operator function name: json_tuple Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6 Select Operator expressions: expr: _col0 @@ -418,9 +424,8 @@ STAGE PLANS: alias: a Lateral View Forward Select Operator - SELECT * : (no compute) Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col2, _col3, _col4, _col5, _col6 Filter Operator predicate: expr: _col2 is not null @@ -469,7 +474,7 @@ STAGE PLANS: UDTF Operator function name: json_tuple Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col2, _col3, _col4, _col5, _col6 Filter Operator predicate: expr: _col2 is not null diff --git ql/src/test/results/clientpositive/udtf_parse_url_tuple.q.out ql/src/test/results/clientpositive/udtf_parse_url_tuple.q.out index 0775703..ec127db 100644 --- ql/src/test/results/clientpositive/udtf_parse_url_tuple.q.out +++ ql/src/test/results/clientpositive/udtf_parse_url_tuple.q.out @@ -84,9 +84,12 @@ STAGE PLANS: alias: a Lateral View Forward Select Operator - SELECT * : (no compute) + expressions: + expr: key + type: string + outputColumnNames: key Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 Select Operator expressions: expr: _col0 @@ -163,7 +166,7 @@ STAGE PLANS: UDTF Operator function name: parse_url_tuple Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 Select Operator expressions: expr: _col0 @@ -376,9 +379,12 @@ STAGE PLANS: alias: a Lateral View Forward Select Operator - SELECT * : (no compute) + expressions: + expr: key + type: string + outputColumnNames: key Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 Select Operator expressions: expr: _col0 @@ -449,7 +455,7 @@ STAGE PLANS: UDTF Operator function name: parse_url_tuple Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 Select Operator expressions: expr: _col0 @@ -542,9 +548,8 @@ STAGE PLANS: alias: a Lateral View Forward Select Operator - SELECT * : (no compute) Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + outputColumnNames: _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 Filter Operator predicate: expr: _col10 is not null @@ -601,7 +606,7 @@ STAGE PLANS: UDTF Operator function name: parse_url_tuple Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + outputColumnNames: _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 Filter Operator predicate: expr: _col10 is not null diff --git ql/src/test/results/clientpositive/udtf_stack.q.out ql/src/test/results/clientpositive/udtf_stack.q.out index 0164e82..fe57cd1 100644 --- ql/src/test/results/clientpositive/udtf_stack.q.out +++ ql/src/test/results/clientpositive/udtf_stack.q.out @@ -23,9 +23,8 @@ STAGE PLANS: alias: src Lateral View Forward Select Operator - SELECT * : (no compute) Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col2, _col3 Select Operator expressions: expr: _col2 @@ -54,7 +53,7 @@ STAGE PLANS: UDTF Operator function name: stack Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col2, _col3 Select Operator expressions: expr: _col2 @@ -95,9 +94,8 @@ STAGE PLANS: alias: src Lateral View Forward Select Operator - SELECT * : (no compute) Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col2, _col3 Select Operator expressions: expr: _col2 @@ -128,7 +126,7 @@ STAGE PLANS: UDTF Operator function name: stack Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col2, _col3 Select Operator expressions: expr: _col2 diff --git ql/src/test/results/clientpositive/union_lateralview.q.out ql/src/test/results/clientpositive/union_lateralview.q.out index 54bfeb7..0e98b01 100644 --- ql/src/test/results/clientpositive/union_lateralview.q.out +++ ql/src/test/results/clientpositive/union_lateralview.q.out @@ -83,9 +83,14 @@ STAGE PLANS: Union Lateral View Forward Select Operator - SELECT * : (no compute) + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + outputColumnNames: _col0, _col1 Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Select Operator expressions: expr: _col3 @@ -117,7 +122,7 @@ STAGE PLANS: UDTF Operator function name: explode Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Select Operator expressions: expr: _col3 @@ -156,9 +161,14 @@ STAGE PLANS: Union Lateral View Forward Select Operator - SELECT * : (no compute) + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + outputColumnNames: _col0, _col1 Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Select Operator expressions: expr: _col3 @@ -190,7 +200,7 @@ STAGE PLANS: UDTF Operator function name: explode Lateral View Join Operator - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col3 Select Operator expressions: expr: _col3