diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 5f9232135f..387dde774d 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -1122,6 +1122,7 @@ spark.query.files=add_part_multiple.q, \ join_vc.q, \ join_view.q, \ lateral_view_explode2.q, \ + lateral_view_multi_lateralviews.q, \ leftsemijoin.q, \ leftsemijoin_mr.q, \ limit_pushdown.q, \ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java index 9bea4ddac9..604c8aee15 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.Stack; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; @@ -617,17 +618,28 @@ private static boolean isSame(List list1, List list2 } @SuppressWarnings("unchecked") - public static T getChildOperator(Operator op, Class klazz) throws SemanticException { - if (klazz.isInstance(op)) { - return (T) op; - } - List> childOperators = op.getChildOperators(); - for (Operator childOp : childOperators) { - T result = getChildOperator(childOp, klazz); - if (result != null) { - return result; + public static T getChildOperator(Operator root, Class klazz) throws SemanticException { + if (root == null) return null; + + HashSet> visited = new HashSet>(); + Stack> stack = new Stack>(); + stack.push(root); + visited.add(root); + + while (!stack.isEmpty()) { + Operator op = stack.pop(); + if (klazz.isInstance(op)) { + return (T) op; + } + List> childOperators = op.getChildOperators(); + for (Operator childOp : childOperators) { + if (!visited.contains(childOp)) { + stack.push(childOp); + visited.add(childOp); + } } } + return null; } diff --git a/ql/src/test/queries/clientpositive/lateral_view_multi_lateralviews.q b/ql/src/test/queries/clientpositive/lateral_view_multi_lateralviews.q new file mode 100644 index 0000000000..29e026a83b --- /dev/null +++ b/ql/src/test/queries/clientpositive/lateral_view_multi_lateralviews.q @@ -0,0 +1,21 @@ +CREATE TABLE t1(x5 STRUCT>>> >); +INSERT INTO t1 SELECT NAMED_STRUCT('x4', NAMED_STRUCT('x3', ARRAY(NAMED_STRUCT('x1', 'x1_1', 'x2', ARRAY('x2_1', 'x2_2'))))); +SELECT c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16 +FROM t1 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c1 +LATERAL VIEW EXPLODE(c1.x2) lv as c2 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c3 +LATERAL VIEW EXPLODE(c1.x2) lv as c4 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c5 +LATERAL VIEW EXPLODE(c1.x2) lv as c6 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c7 +LATERAL VIEW EXPLODE(c1.x2) lv as c8 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c9 +LATERAL VIEW EXPLODE(c1.x2) lv as c10 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c11 +LATERAL VIEW EXPLODE(c1.x2) lv as c12 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c13 +LATERAL VIEW EXPLODE(c1.x2) lv as c14 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c15 +LATERAL VIEW EXPLODE(c1.x2) lv as c16 +LIMIT 1; diff --git a/ql/src/test/results/clientpositive/lateral_view_multi_lateralviews.q.out b/ql/src/test/results/clientpositive/lateral_view_multi_lateralviews.q.out new file mode 100644 index 0000000000..b9c827fe0c --- /dev/null +++ b/ql/src/test/results/clientpositive/lateral_view_multi_lateralviews.q.out @@ -0,0 +1,62 @@ +PREHOOK: query: CREATE TABLE t1(x5 STRUCT>>> >) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: CREATE TABLE t1(x5 STRUCT>>> >) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: INSERT INTO t1 SELECT NAMED_STRUCT('x4', NAMED_STRUCT('x3', ARRAY(NAMED_STRUCT('x1', 'x1_1', 'x2', ARRAY('x2_1', 'x2_2'))))) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@t1 +POSTHOOK: query: INSERT INTO t1 SELECT NAMED_STRUCT('x4', NAMED_STRUCT('x3', ARRAY(NAMED_STRUCT('x1', 'x1_1', 'x2', ARRAY('x2_1', 'x2_2'))))) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@t1 +POSTHOOK: Lineage: t1.x5 EXPRESSION [] +PREHOOK: query: SELECT c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16 +FROM t1 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c1 +LATERAL VIEW EXPLODE(c1.x2) lv as c2 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c3 +LATERAL VIEW EXPLODE(c1.x2) lv as c4 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c5 +LATERAL VIEW EXPLODE(c1.x2) lv as c6 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c7 +LATERAL VIEW EXPLODE(c1.x2) lv as c8 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c9 +LATERAL VIEW EXPLODE(c1.x2) lv as c10 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c11 +LATERAL VIEW EXPLODE(c1.x2) lv as c12 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c13 +LATERAL VIEW EXPLODE(c1.x2) lv as c14 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c15 +LATERAL VIEW EXPLODE(c1.x2) lv as c16 +LIMIT 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16 +FROM t1 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c1 +LATERAL VIEW EXPLODE(c1.x2) lv as c2 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c3 +LATERAL VIEW EXPLODE(c1.x2) lv as c4 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c5 +LATERAL VIEW EXPLODE(c1.x2) lv as c6 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c7 +LATERAL VIEW EXPLODE(c1.x2) lv as c8 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c9 +LATERAL VIEW EXPLODE(c1.x2) lv as c10 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c11 +LATERAL VIEW EXPLODE(c1.x2) lv as c12 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c13 +LATERAL VIEW EXPLODE(c1.x2) lv as c14 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c15 +LATERAL VIEW EXPLODE(c1.x2) lv as c16 +LIMIT 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +#### A masked pattern was here #### +{"x1":"x1_1","x2":["x2_1","x2_2"]} x2_1 {"x1":"x1_1","x2":["x2_1","x2_2"]} x2_1 {"x1":"x1_1","x2":["x2_1","x2_2"]} x2_1 {"x1":"x1_1","x2":["x2_1","x2_2"]} x2_1 {"x1":"x1_1","x2":["x2_1","x2_2"]} x2_1 {"x1":"x1_1","x2":["x2_1","x2_2"]} x2_1 {"x1":"x1_1","x2":["x2_1","x2_2"]} x2_1 {"x1":"x1_1","x2":["x2_1","x2_2"]} x2_1 diff --git a/ql/src/test/results/clientpositive/spark/lateral_view_multi_lateralviews.q.out b/ql/src/test/results/clientpositive/spark/lateral_view_multi_lateralviews.q.out new file mode 100644 index 0000000000..b9c827fe0c --- /dev/null +++ b/ql/src/test/results/clientpositive/spark/lateral_view_multi_lateralviews.q.out @@ -0,0 +1,62 @@ +PREHOOK: query: CREATE TABLE t1(x5 STRUCT>>> >) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: CREATE TABLE t1(x5 STRUCT>>> >) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: INSERT INTO t1 SELECT NAMED_STRUCT('x4', NAMED_STRUCT('x3', ARRAY(NAMED_STRUCT('x1', 'x1_1', 'x2', ARRAY('x2_1', 'x2_2'))))) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@t1 +POSTHOOK: query: INSERT INTO t1 SELECT NAMED_STRUCT('x4', NAMED_STRUCT('x3', ARRAY(NAMED_STRUCT('x1', 'x1_1', 'x2', ARRAY('x2_1', 'x2_2'))))) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@t1 +POSTHOOK: Lineage: t1.x5 EXPRESSION [] +PREHOOK: query: SELECT c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16 +FROM t1 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c1 +LATERAL VIEW EXPLODE(c1.x2) lv as c2 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c3 +LATERAL VIEW EXPLODE(c1.x2) lv as c4 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c5 +LATERAL VIEW EXPLODE(c1.x2) lv as c6 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c7 +LATERAL VIEW EXPLODE(c1.x2) lv as c8 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c9 +LATERAL VIEW EXPLODE(c1.x2) lv as c10 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c11 +LATERAL VIEW EXPLODE(c1.x2) lv as c12 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c13 +LATERAL VIEW EXPLODE(c1.x2) lv as c14 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c15 +LATERAL VIEW EXPLODE(c1.x2) lv as c16 +LIMIT 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16 +FROM t1 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c1 +LATERAL VIEW EXPLODE(c1.x2) lv as c2 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c3 +LATERAL VIEW EXPLODE(c1.x2) lv as c4 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c5 +LATERAL VIEW EXPLODE(c1.x2) lv as c6 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c7 +LATERAL VIEW EXPLODE(c1.x2) lv as c8 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c9 +LATERAL VIEW EXPLODE(c1.x2) lv as c10 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c11 +LATERAL VIEW EXPLODE(c1.x2) lv as c12 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c13 +LATERAL VIEW EXPLODE(c1.x2) lv as c14 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c15 +LATERAL VIEW EXPLODE(c1.x2) lv as c16 +LIMIT 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +#### A masked pattern was here #### +{"x1":"x1_1","x2":["x2_1","x2_2"]} x2_1 {"x1":"x1_1","x2":["x2_1","x2_2"]} x2_1 {"x1":"x1_1","x2":["x2_1","x2_2"]} x2_1 {"x1":"x1_1","x2":["x2_1","x2_2"]} x2_1 {"x1":"x1_1","x2":["x2_1","x2_2"]} x2_1 {"x1":"x1_1","x2":["x2_1","x2_2"]} x2_1 {"x1":"x1_1","x2":["x2_1","x2_2"]} x2_1 {"x1":"x1_1","x2":["x2_1","x2_2"]} x2_1