Index: contrib/src/test/results/clientpositive/serde_typedbytes.q.out =================================================================== --- contrib/src/test/results/clientpositive/serde_typedbytes.q.out (revision 1365861) +++ contrib/src/test/results/clientpositive/serde_typedbytes.q.out (working copy) @@ -34,11 +34,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -82,7 +84,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -117,7 +119,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM ( FROM src SELECT TRANSFORM(src.key, src.value) ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe' Index: contrib/src/test/results/clientpositive/serde_typedbytes2.q.out =================================================================== --- contrib/src/test/results/clientpositive/serde_typedbytes2.q.out (revision 1365861) +++ contrib/src/test/results/clientpositive/serde_typedbytes2.q.out (working copy) @@ -34,11 +34,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -75,7 +77,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -110,7 +112,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM ( FROM src SELECT TRANSFORM(cast(src.key as smallint), src.value) ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe' Index: contrib/src/test/results/clientpositive/serde_typedbytes3.q.out =================================================================== --- contrib/src/test/results/clientpositive/serde_typedbytes3.q.out (revision 1365861) +++ contrib/src/test/results/clientpositive/serde_typedbytes3.q.out (working copy) @@ -34,11 +34,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -75,7 +77,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -110,7 +112,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM ( FROM src SELECT TRANSFORM(cast(src.key as smallint), src.value) ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe' Index: contrib/src/test/results/clientpositive/serde_typedbytes5.q.out =================================================================== --- contrib/src/test/results/clientpositive/serde_typedbytes5.q.out (revision 1365861) +++ contrib/src/test/results/clientpositive/serde_typedbytes5.q.out (working copy) @@ -34,11 +34,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -82,7 +84,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -117,7 +119,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM ( FROM src SELECT TRANSFORM(src.key, src.value) ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe' Index: ql/src/test/results/clientpositive/merge4.q.out =================================================================== --- ql/src/test/results/clientpositive/merge4.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/merge4.q.out (working copy) @@ -14,11 +14,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -45,7 +47,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -83,7 +85,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table nzhang_part partition (ds='2010-08-15', hr) select key, value, hr from srcpart where ds='2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -1128,11 +1149,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -1157,7 +1180,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -1195,7 +1218,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table nzhang_part partition (ds='2010-08-15', hr=11) select key, value from srcpart where ds='2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -2752,16 +2794,18 @@ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_TABLE_OR_COL hr))) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2008-04-08')))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR '1' key) (TOK_SELEXPR '1' value) (TOK_SELEXPR 'file,' hr)) (TOK_LIMIT 1)))) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME nzhang_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2010-08-15') (TOK_PARTVAL hr)))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-2 depends on stages: Stage-7 - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-9 is a root stage + Stage-2 depends on stages: Stage-9 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 + Stage-6 + Stage-7 depends on stages: Stage-6 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-9 Map Reduce Alias -> Map Operator Tree: null-subquery2:s-subquery2:src @@ -2851,7 +2895,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -2889,7 +2933,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table nzhang_part partition (ds='2010-08-15', hr) select * from ( select key, value, hr from srcpart where ds='2008-04-08' Index: ql/src/test/results/clientpositive/index_auto_update.q.out =================================================================== --- ql/src/test/results/clientpositive/index_auto_update.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/index_auto_update.q.out (working copy) @@ -57,9 +57,9 @@ STAGE DEPENDENCIES: Stage-4 is a root stage - Stage-8 depends on stages: Stage-4 , consists of Stage-7, Stage-6 + Stage-10 depends on stages: Stage-4 , consists of Stage-7, Stage-6, Stage-8 Stage-7 - Stage-0 depends on stages: Stage-7, Stage-6 + Stage-0 depends on stages: Stage-7, Stage-6, Stage-9 Stage-1 depends on stages: Stage-0 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 @@ -67,6 +67,8 @@ Stage-3 depends on stages: Stage-1 Stage-5 depends on stages: Stage-0 Stage-6 + Stage-8 + Stage-9 depends on stages: Stage-8 STAGE PLANS: Stage: Stage-4 @@ -91,7 +93,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.temp - Stage: Stage-8 + Stage: Stage-10 Conditional Operator Stage: Stage-7 @@ -215,7 +217,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.temp + Stage: Stage-8 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.temp + Stage: Stage-9 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE temp SELECT * FROM src PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -253,11 +274,13 @@ STAGE DEPENDENCIES: Stage-3 is a root stage - Stage-6 depends on stages: Stage-3 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-3 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-2 depends on stages: Stage-5, Stage-4 + Stage-2 depends on stages: Stage-5, Stage-4, Stage-7 Stage-1 depends on stages: Stage-2 Stage-4 + Stage-6 + Stage-7 depends on stages: Stage-6 Stage-0 is a root stage STAGE PLANS: @@ -288,7 +311,7 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -341,6 +364,23 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + Stage: Stage-0 Fetch Operator limit: -1 Index: ql/src/test/results/clientpositive/union28.q.out =================================================================== --- ql/src/test/results/clientpositive/union28.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/union28.q.out (working copy) @@ -33,18 +33,20 @@ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))))) (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_GROUPBY (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_GROUPBY (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value))))) subq)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME union_subq_union))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-8 depends on stages: Stage-7, Stage-9 - Stage-2 depends on stages: Stage-8 - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-9 is a root stage + Stage-10 depends on stages: Stage-9, Stage-11 + Stage-2 depends on stages: Stage-10 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 - Stage-9 is a root stage + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-11 is a root stage STAGE PLANS: - Stage: Stage-7 + Stage: Stage-9 Map Reduce Alias -> Map Operator Tree: null-subquery2:a-subquery2-subquery1:subq-subquery1:src @@ -112,7 +114,7 @@ input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - Stage: Stage-8 + Stage: Stage-10 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -210,7 +212,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -245,9 +247,28 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union - Stage: Stage-9 + Stage: Stage-6 Map Reduce Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-11 + Map Reduce + Alias -> Map Operator Tree: null-subquery2:a-subquery2-subquery2:subq-subquery2:src TableScan alias: src Index: ql/src/test/results/clientpositive/input6.q.out =================================================================== --- ql/src/test/results/clientpositive/input6.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input6.q.out (working copy) @@ -16,11 +16,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -49,7 +51,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -84,7 +86,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM src1 INSERT OVERWRITE TABLE dest1 SELECT src1.key, src1.value WHERE src1.key is null PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/insert_into4.q.out =================================================================== --- ql/src/test/results/clientpositive/insert_into4.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/insert_into4.q.out (working copy) @@ -225,11 +225,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -254,7 +256,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.insert_into4b - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -289,7 +291,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.insert_into4b + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into4b + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT INTO TABLE insert_into4b SELECT * FROM insert_into4a PREHOOK: type: QUERY PREHOOK: Input: default@insert_into4a Index: ql/src/test/results/clientpositive/pcr.q.out =================================================================== --- ql/src/test/results/clientpositive/pcr.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/pcr.q.out (working copy) @@ -3639,16 +3639,20 @@ STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 - Stage-10 depends on stages: Stage-2 , consists of Stage-9, Stage-8 - Stage-9 - Stage-1 depends on stages: Stage-9, Stage-8 - Stage-7 depends on stages: Stage-1 - Stage-8 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-14 depends on stages: Stage-2 , consists of Stage-11, Stage-10, Stage-12 + Stage-11 + Stage-1 depends on stages: Stage-11, Stage-10, Stage-13 + Stage-9 depends on stages: Stage-1 + Stage-10 + Stage-12 + Stage-13 depends on stages: Stage-12 STAGE PLANS: Stage: Stage-2 @@ -3771,7 +3775,7 @@ name: default.pcr_t1 name: default.pcr_t1 - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -3870,10 +3874,80 @@ name: default.pcr_t2 name: default.pcr_t2 - Stage: Stage-10 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.pcr_t2 + serialization.ddl struct pcr_t2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.pcr_t2 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10004 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.pcr_t2 + serialization.ddl struct pcr_t2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.pcr_t2 + serialization.ddl struct pcr_t2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.pcr_t2 + name: default.pcr_t2 + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-14 Conditional Operator - Stage: Stage-9 + Stage: Stage-11 Move Operator files: hdfs directory: true @@ -3901,11 +3975,11 @@ name: default.pcr_t3 #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-9 Stats-Aggr Operator #### A masked pattern was here #### - Stage: Stage-8 + Stage: Stage-10 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -3969,7 +4043,77 @@ name: default.pcr_t3 name: default.pcr_t3 + Stage: Stage-12 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.pcr_t3 + serialization.ddl struct pcr_t3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.pcr_t3 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10005 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.pcr_t3 + serialization.ddl struct pcr_t3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.pcr_t3 + serialization.ddl struct pcr_t3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.pcr_t3 + name: default.pcr_t3 + Stage: Stage-13 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: from pcr_t1 insert overwrite table pcr_t2 select key, value where ds='2000-04-08' insert overwrite table pcr_t3 select key, value where ds='2000-04-08' @@ -4023,16 +4167,20 @@ STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 - Stage-10 depends on stages: Stage-2 , consists of Stage-9, Stage-8 - Stage-9 - Stage-1 depends on stages: Stage-9, Stage-8 - Stage-7 depends on stages: Stage-1 - Stage-8 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-14 depends on stages: Stage-2 , consists of Stage-11, Stage-10, Stage-12 + Stage-11 + Stage-1 depends on stages: Stage-11, Stage-10, Stage-13 + Stage-9 depends on stages: Stage-1 + Stage-10 + Stage-12 + Stage-13 depends on stages: Stage-12 STAGE PLANS: Stage: Stage-2 @@ -4175,7 +4323,7 @@ name: default.pcr_t1 name: default.pcr_t1 - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -4294,10 +4442,95 @@ name: default.pcr_t2 name: default.pcr_t2 - Stage: Stage-10 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.pcr_t2 + numFiles 1 + numPartitions 0 + numRows 20 + rawDataSize 160 + serialization.ddl struct pcr_t2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 180 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.pcr_t2 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10004 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.pcr_t2 + numFiles 1 + numPartitions 0 + numRows 20 + rawDataSize 160 + serialization.ddl struct pcr_t2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 180 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.pcr_t2 + numFiles 1 + numPartitions 0 + numRows 20 + rawDataSize 160 + serialization.ddl struct pcr_t2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 180 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.pcr_t2 + name: default.pcr_t2 + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-14 Conditional Operator - Stage: Stage-9 + Stage: Stage-11 Move Operator files: hdfs directory: true @@ -4330,11 +4563,11 @@ name: default.pcr_t3 #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-9 Stats-Aggr Operator #### A masked pattern was here #### - Stage: Stage-8 + Stage: Stage-10 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -4413,7 +4646,92 @@ name: default.pcr_t3 name: default.pcr_t3 + Stage: Stage-12 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.pcr_t3 + numFiles 1 + numPartitions 0 + numRows 20 + rawDataSize 160 + serialization.ddl struct pcr_t3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 180 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.pcr_t3 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10005 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.pcr_t3 + numFiles 1 + numPartitions 0 + numRows 20 + rawDataSize 160 + serialization.ddl struct pcr_t3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 180 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.pcr_t3 + numFiles 1 + numPartitions 0 + numRows 20 + rawDataSize 160 + serialization.ddl struct pcr_t3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 180 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.pcr_t3 + name: default.pcr_t3 + Stage: Stage-13 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: from pcr_t1 insert overwrite table pcr_t2 select key, value where ds='2000-04-08' and key=2 insert overwrite table pcr_t3 select key, value where ds='2000-04-08' and key=3 Index: ql/src/test/results/clientpositive/index_auto_partitioned.q.out =================================================================== --- ql/src/test/results/clientpositive/index_auto_partitioned.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/index_auto_partitioned.q.out (working copy) @@ -40,11 +40,13 @@ STAGE DEPENDENCIES: Stage-3 is a root stage - Stage-6 depends on stages: Stage-3 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-3 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-2 depends on stages: Stage-5, Stage-4 + Stage-2 depends on stages: Stage-5, Stage-4, Stage-7 Stage-1 depends on stages: Stage-2 Stage-4 + Stage-6 + Stage-7 depends on stages: Stage-6 Stage-0 is a root stage STAGE PLANS: @@ -75,7 +77,7 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -141,6 +143,23 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + Stage: Stage-0 Fetch Operator limit: -1 Index: ql/src/test/results/clientpositive/index_auto_file_format.q.out =================================================================== --- ql/src/test/results/clientpositive/index_auto_file_format.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/index_auto_file_format.q.out (working copy) @@ -28,11 +28,13 @@ STAGE DEPENDENCIES: Stage-3 is a root stage - Stage-6 depends on stages: Stage-3 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-3 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-2 depends on stages: Stage-5, Stage-4 + Stage-2 depends on stages: Stage-5, Stage-4, Stage-7 Stage-1 depends on stages: Stage-2 Stage-4 + Stage-6 + Stage-7 depends on stages: Stage-6 Stage-0 is a root stage STAGE PLANS: @@ -63,7 +65,7 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -129,6 +131,23 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + Stage: Stage-0 Fetch Operator limit: -1 @@ -160,11 +179,13 @@ STAGE DEPENDENCIES: Stage-3 is a root stage - Stage-6 depends on stages: Stage-3 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-3 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-2 depends on stages: Stage-5, Stage-4 + Stage-2 depends on stages: Stage-5, Stage-4, Stage-7 Stage-1 depends on stages: Stage-2 Stage-4 + Stage-6 + Stage-7 depends on stages: Stage-6 Stage-0 is a root stage STAGE PLANS: @@ -195,7 +216,7 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -261,6 +282,23 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + Stage: Stage-0 Fetch Operator limit: -1 Index: ql/src/test/results/clientpositive/input_part2.q.out =================================================================== --- ql/src/test/results/clientpositive/input_part2.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input_part2.q.out (working copy) @@ -23,16 +23,20 @@ STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 - Stage-10 depends on stages: Stage-2 , consists of Stage-9, Stage-8 - Stage-9 - Stage-1 depends on stages: Stage-9, Stage-8 - Stage-7 depends on stages: Stage-1 - Stage-8 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-14 depends on stages: Stage-2 , consists of Stage-11, Stage-10, Stage-12 + Stage-11 + Stage-1 depends on stages: Stage-11, Stage-10, Stage-13 + Stage-9 depends on stages: Stage-1 + Stage-10 + Stage-12 + Stage-13 depends on stages: Stage-12 STAGE PLANS: Stage: Stage-2 @@ -223,7 +227,7 @@ name: default.srcpart name: default.srcpart - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -322,10 +326,80 @@ name: default.dest1 name: default.dest1 - Stage: Stage-10 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,hr,ds + columns.types int:string:string:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value, string hr, string ds} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10004 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,hr,ds + columns.types int:string:string:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value, string hr, string ds} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,hr,ds + columns.types int:string:string:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value, string hr, string ds} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + name: default.dest1 + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-14 Conditional Operator - Stage: Stage-9 + Stage: Stage-11 Move Operator files: hdfs directory: true @@ -353,11 +427,11 @@ name: default.dest2 #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-9 Stats-Aggr Operator #### A masked pattern was here #### - Stage: Stage-8 + Stage: Stage-10 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -421,7 +495,77 @@ name: default.dest2 name: default.dest2 + Stage: Stage-12 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,hr,ds + columns.types int:string:string:string +#### A masked pattern was here #### + name default.dest2 + serialization.ddl struct dest2 { i32 key, string value, string hr, string ds} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10005 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,hr,ds + columns.types int:string:string:string +#### A masked pattern was here #### + name default.dest2 + serialization.ddl struct dest2 { i32 key, string value, string hr, string ds} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,hr,ds + columns.types int:string:string:string +#### A masked pattern was here #### + name default.dest2 + serialization.ddl struct dest2 { i32 key, string value, string hr, string ds} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2 + name: default.dest2 + Stage: Stage-13 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM srcpart INSERT OVERWRITE TABLE dest1 SELECT srcpart.key, srcpart.value, srcpart.hr, srcpart.ds WHERE srcpart.key < 100 and srcpart.ds = '2008-04-08' and srcpart.hr = '12' INSERT OVERWRITE TABLE dest2 SELECT srcpart.key, srcpart.value, srcpart.hr, srcpart.ds WHERE srcpart.key < 100 and srcpart.ds = '2008-04-09' and srcpart.hr = '12' Index: ql/src/test/results/clientpositive/rcfile_merge2.q.out =================================================================== --- ql/src/test/results/clientpositive/rcfile_merge2.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/rcfile_merge2.q.out (working copy) @@ -26,11 +26,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -70,7 +72,7 @@ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.rcfile_merge2a - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -99,7 +101,16 @@ Stage: Stage-3 Block level merge + Stage: Stage-5 + Block level merge + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE rcfile_merge2a PARTITION (one='1', two, three) SELECT key, value, PMOD(HASH(key), 10) as two, PMOD(HASH(value), 10) as three Index: ql/src/test/results/clientpositive/sample4.q.out =================================================================== --- ql/src/test/results/clientpositive/sample4.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/sample4.q.out (working copy) @@ -20,11 +20,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -109,7 +111,7 @@ name: default.srcbucket name: default.srcbucket - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -208,7 +210,77 @@ name: default.dest1 name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) s PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/input11.q.out =================================================================== --- ql/src/test/results/clientpositive/input11.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input11.q.out (working copy) @@ -16,11 +16,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -56,7 +58,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -91,7 +93,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, src.value WHERE src.key < 100 PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/input34.q.out =================================================================== --- ql/src/test/results/clientpositive/input34.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input34.q.out (working copy) @@ -26,11 +26,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -74,7 +76,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -109,7 +111,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM ( FROM src SELECT TRANSFORM(src.key, src.value) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' Index: ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out =================================================================== --- ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out (working copy) @@ -243,17 +243,21 @@ STAGE DEPENDENCIES: Stage-5 is a root stage - Stage-8 depends on stages: Stage-5 , consists of Stage-7, Stage-6 + Stage-10 depends on stages: Stage-5 , consists of Stage-7, Stage-6, Stage-8 Stage-7 - Stage-4 depends on stages: Stage-7, Stage-6 - Stage-1 depends on stages: Stage-4, Stage-9 + Stage-4 depends on stages: Stage-7, Stage-6, Stage-9 + Stage-1 depends on stages: Stage-4, Stage-11 Stage-2 depends on stages: Stage-1 Stage-6 - Stage-10 is a root stage - Stage-13 depends on stages: Stage-10 , consists of Stage-12, Stage-11 - Stage-12 - Stage-9 depends on stages: Stage-12, Stage-11 - Stage-11 + Stage-8 + Stage-9 depends on stages: Stage-8 + Stage-12 is a root stage + Stage-17 depends on stages: Stage-12 , consists of Stage-14, Stage-13, Stage-15 + Stage-14 + Stage-11 depends on stages: Stage-14, Stage-13, Stage-16 + Stage-13 + Stage-15 + Stage-16 depends on stages: Stage-15 Stage-0 is a root stage STAGE PLANS: @@ -284,7 +288,7 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - Stage: Stage-8 + Stage: Stage-10 Conditional Operator Stage: Stage-7 @@ -406,9 +410,26 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - Stage: Stage-10 + Stage: Stage-8 Map Reduce Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-9 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-12 + Map Reduce + Alias -> Map Operator Tree: default__src_src_index__ TableScan alias: default__src_src_index__ @@ -433,22 +454,22 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - Stage: Stage-13 + Stage: Stage-17 Conditional Operator - Stage: Stage-12 + Stage: Stage-14 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-9 + Stage: Stage-11 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-11 + Stage: Stage-13 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -459,6 +480,23 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + Stage: Stage-15 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-16 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + Stage: Stage-0 Fetch Operator limit: -1 Index: ql/src/test/results/clientpositive/merge1.q.out =================================================================== --- ql/src/test/results/clientpositive/merge1.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/merge1.q.out (working copy) @@ -16,11 +16,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -88,7 +90,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -123,7 +125,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table dest1 select key, count(1) from src group by key PREHOOK: type: QUERY @@ -522,11 +543,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -549,7 +572,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -584,7 +607,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table dest1 select key from test_src PREHOOK: type: QUERY PREHOOK: Input: default@test_src@ds=101 @@ -620,11 +662,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -647,7 +691,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -682,7 +726,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table dest1 select key from test_src PREHOOK: type: QUERY PREHOOK: Input: default@test_src@ds=101 Index: ql/src/test/results/clientpositive/lineage1.q.out =================================================================== --- ql/src/test/results/clientpositive/lineage1.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/lineage1.q.out (working copy) @@ -34,13 +34,15 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-7 - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-2 depends on stages: Stage-1, Stage-9 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 - Stage-7 is a root stage + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-9 is a root stage STAGE PLANS: Stage: Stage-1 @@ -153,7 +155,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_l1 - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -188,7 +190,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_l1 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_l1 + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-9 Map Reduce Alias -> Map Operator Tree: null-subquery2:j-subquery2:p2 Index: ql/src/test/results/clientpositive/input8.q.out =================================================================== --- ql/src/test/results/clientpositive/input8.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input8.q.out (working copy) @@ -16,11 +16,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -56,7 +58,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -91,7 +93,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM src1 INSERT OVERWRITE TABLE dest1 SELECT 4 + NULL, src1.key - NULL, NULL + NULL PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/insert_into6.q.out =================================================================== --- ql/src/test/results/clientpositive/insert_into6.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/insert_into6.q.out (working copy) @@ -144,11 +144,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -175,7 +177,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.insert_into6b - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -212,7 +214,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.insert_into6b + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into6b + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT INTO TABLE insert_into6b PARTITION (ds) SELECT * FROM insert_into6a PREHOOK: type: QUERY PREHOOK: Input: default@insert_into6a@ds=1 Index: ql/src/test/results/clientpositive/merge_dynamic_partition5.q.out =================================================================== --- ql/src/test/results/clientpositive/merge_dynamic_partition5.q.out (revision 0) +++ ql/src/test/results/clientpositive/merge_dynamic_partition5.q.out (working copy) @@ -0,0 +1,226 @@ +PREHOOK: query: -- this is to test the case where some dynamic partitions are merged and some are moved + +create table srcpart_merge_dp like srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- this is to test the case where some dynamic partitions are merged and some are moved + +create table srcpart_merge_dp like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@srcpart_merge_dp +PREHOOK: query: create table srcpart_merge_dp_rc like srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table srcpart_merge_dp_rc like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@srcpart_merge_dp_rc +PREHOOK: query: alter table srcpart_merge_dp_rc set fileformat RCFILE +PREHOOK: type: ALTERTABLE_FILEFORMAT +PREHOOK: Input: default@srcpart_merge_dp_rc +PREHOOK: Output: default@srcpart_merge_dp_rc +POSTHOOK: query: alter table srcpart_merge_dp_rc set fileformat RCFILE +POSTHOOK: type: ALTERTABLE_FILEFORMAT +POSTHOOK: Input: default@srcpart_merge_dp_rc +POSTHOOK: Output: default@srcpart_merge_dp_rc +PREHOOK: query: create table merge_dynamic_part like srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table merge_dynamic_part like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@merge_dynamic_part +PREHOOK: query: alter table merge_dynamic_part set fileformat RCFILE +PREHOOK: type: ALTERTABLE_FILEFORMAT +PREHOOK: Input: default@merge_dynamic_part +PREHOOK: Output: default@merge_dynamic_part +POSTHOOK: query: alter table merge_dynamic_part set fileformat RCFILE +POSTHOOK: type: ALTERTABLE_FILEFORMAT +POSTHOOK: Input: default@merge_dynamic_part +POSTHOOK: Output: default@merge_dynamic_part +PREHOOK: query: load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11) +PREHOOK: type: LOAD +PREHOOK: Output: default@srcpart_merge_dp +POSTHOOK: query: load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11) +POSTHOOK: type: LOAD +POSTHOOK: Output: default@srcpart_merge_dp +POSTHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +PREHOOK: query: load data local inpath '../data/files/srcbucket21.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11) +PREHOOK: type: LOAD +PREHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +POSTHOOK: query: load data local inpath '../data/files/srcbucket21.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11) +POSTHOOK: type: LOAD +POSTHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +PREHOOK: query: load data local inpath '../data/files/srcbucket22.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11) +PREHOOK: type: LOAD +PREHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +POSTHOOK: query: load data local inpath '../data/files/srcbucket22.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11) +POSTHOOK: type: LOAD +POSTHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +PREHOOK: query: load data local inpath '../data/files/srcbucket23.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11) +PREHOOK: type: LOAD +PREHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +POSTHOOK: query: load data local inpath '../data/files/srcbucket23.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11) +POSTHOOK: type: LOAD +POSTHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +PREHOOK: query: load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=12) +PREHOOK: type: LOAD +PREHOOK: Output: default@srcpart_merge_dp +POSTHOOK: query: load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=12) +POSTHOOK: type: LOAD +POSTHOOK: Output: default@srcpart_merge_dp +POSTHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=12 +PREHOOK: query: insert overwrite table srcpart_merge_dp_rc partition (ds = '2008-04-08', hr) +select key, value, hr from srcpart_merge_dp where ds = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart_merge_dp@ds=2008-04-08/hr=12 +PREHOOK: Output: default@srcpart_merge_dp_rc@ds=2008-04-08 +POSTHOOK: query: insert overwrite table srcpart_merge_dp_rc partition (ds = '2008-04-08', hr) +select key, value, hr from srcpart_merge_dp where ds = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart_merge_dp@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@srcpart_merge_dp_rc@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@srcpart_merge_dp_rc@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +insert overwrite table merge_dynamic_part partition (ds = '2008-04-08', hr) +select key, value, if(key % 100 == 0, 'a1', 'b1') as hr from srcpart_merge_dp_rc where ds = '2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table merge_dynamic_part partition (ds = '2008-04-08', hr) +select key, value, if(key % 100 == 0, 'a1', 'b1') as hr from srcpart_merge_dp_rc where ds = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart_merge_dp_rc))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME merge_dynamic_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr)))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION if (== (% (TOK_TABLE_OR_COL key) 100) 0) 'a1' 'b1') hr)) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2008-04-08')))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + srcpart_merge_dp_rc + TableScan + alias: srcpart_merge_dp_rc + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + expr: if(((key % 100) = 0), 'a1', 'b1') + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.merge_dynamic_part + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 2008-04-08 + hr + replace: true + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.merge_dynamic_part + + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 + Block level merge + + Stage: Stage-5 + Block level merge + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + +PREHOOK: query: insert overwrite table merge_dynamic_part partition (ds = '2008-04-08', hr) +select key, value, if(key % 100 == 0, 'a1', 'b1') as hr from srcpart_merge_dp_rc where ds = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart_merge_dp_rc@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart_merge_dp_rc@ds=2008-04-08/hr=12 +PREHOOK: Output: default@merge_dynamic_part@ds=2008-04-08 +POSTHOOK: query: insert overwrite table merge_dynamic_part partition (ds = '2008-04-08', hr) +select key, value, if(key % 100 == 0, 'a1', 'b1') as hr from srcpart_merge_dp_rc where ds = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart_merge_dp_rc@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart_merge_dp_rc@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@merge_dynamic_part@ds=2008-04-08/hr=a1 +POSTHOOK: Output: default@merge_dynamic_part@ds=2008-04-08/hr=b1 +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=a1).key SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=a1).value SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=b1).key SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=b1).value SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: show partitions merge_dynamic_part +PREHOOK: type: SHOWPARTITIONS +POSTHOOK: query: show partitions merge_dynamic_part +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=a1).key SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=a1).value SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=b1).key SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=b1).value SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +ds=2008-04-08/hr=a1 +ds=2008-04-08/hr=b1 +PREHOOK: query: select count(*) from merge_dynamic_part +PREHOOK: type: QUERY +PREHOOK: Input: default@merge_dynamic_part@ds=2008-04-08/hr=a1 +PREHOOK: Input: default@merge_dynamic_part@ds=2008-04-08/hr=b1 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from merge_dynamic_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@merge_dynamic_part@ds=2008-04-08/hr=a1 +POSTHOOK: Input: default@merge_dynamic_part@ds=2008-04-08/hr=b1 +#### A masked pattern was here #### +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=a1).key SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=a1).value SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=b1).key SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=b1).value SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +618 Index: ql/src/test/results/clientpositive/udf1.q.out =================================================================== --- ql/src/test/results/clientpositive/udf1.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/udf1.q.out (working copy) @@ -34,11 +34,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -103,7 +105,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -138,7 +140,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT 'a' LIKE '%a%', 'b' LIKE '%a%', 'ab' LIKE '%a%', 'ab' LIKE '%a_', '%_' LIKE '\%\_', 'ab' LIKE '\%\_', 'ab' LIKE '_a%', 'ab' LIKE 'a', '' RLIKE '.*', 'a' RLIKE '[ab]', '' RLIKE '[ab]', 'hadoop' RLIKE '[a-z]*', 'hadoop' RLIKE 'o*', Index: ql/src/test/results/clientpositive/join35.q.out =================================================================== --- ql/src/test/results/clientpositive/join35.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/join35.q.out (working copy) @@ -28,14 +28,16 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-9 depends on stages: Stage-1, Stage-7 - Stage-2 depends on stages: Stage-9 - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-11 depends on stages: Stage-1, Stage-9 + Stage-2 depends on stages: Stage-11 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 - Stage-7 is a root stage + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-9 is a root stage STAGE PLANS: Stage: Stage-1 @@ -145,7 +147,7 @@ GatherStats: false MultiFileSpray: false - Stage: Stage-9 + Stage: Stage-11 Map Reduce Local Work Alias -> Map Local Tables: x @@ -342,7 +344,7 @@ columns.types string,bigint escape.delim \ - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -441,7 +443,77 @@ name: default.dest_j1 name: default.dest_j1 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.types string:string:int +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, i32 val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10003 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.types string:string:int +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, i32 val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.types string:string:int +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, i32 val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + name: default.dest_j1 + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-9 Map Reduce Alias -> Map Operator Tree: null-subquery2:subq1-subquery2:x1 Index: ql/src/test/results/clientpositive/rcfile_merge4.q.out =================================================================== --- ql/src/test/results/clientpositive/rcfile_merge4.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/rcfile_merge4.q.out (working copy) @@ -59,11 +59,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -88,7 +90,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.rcfile_merge3b - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -123,7 +125,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.rcfile_merge3b + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.rcfile_merge3b + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE rcfile_merge3b SELECT key, value FROM rcfile_merge3a PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/join_map_ppr.q.out =================================================================== --- ql/src/test/results/clientpositive/join_map_ppr.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/join_map_ppr.q.out (working copy) @@ -21,16 +21,18 @@ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME src1) x) (TOK_TABREF (TOK_TABNAME src) y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL y) key))) (TOK_TABREF (TOK_TABNAME srcpart) z) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL z) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest_j1))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST x y))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL z) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL y) value))) (TOK_WHERE (and (= (. (TOK_TABLE_OR_COL z) ds) '2008-04-08') (= (. (TOK_TABLE_OR_COL z) hr) 11))))) STAGE DEPENDENCIES: - Stage-8 is a root stage - Stage-1 depends on stages: Stage-8 - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-10 is a root stage + Stage-1 depends on stages: Stage-10 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-8 + Stage: Stage-10 Map Reduce Local Work Alias -> Map Local Tables: x @@ -183,7 +185,7 @@ name: default.srcpart name: default.srcpart - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -282,7 +284,77 @@ name: default.dest_j1 name: default.dest_j1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + name: default.dest_j1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x,y) */ x.key, z.value, y.value FROM src1 x JOIN src y ON (x.key = y.key) @@ -493,16 +565,18 @@ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME src1_copy) x) (TOK_TABREF (TOK_TABNAME src_copy) y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL y) key))) (TOK_TABREF (TOK_TABNAME srcpart) z) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL z) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest_j1))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST x y))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL z) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL y) value))) (TOK_WHERE (and (= (. (TOK_TABLE_OR_COL z) ds) '2008-04-08') (= (. (TOK_TABLE_OR_COL z) hr) 11))))) STAGE DEPENDENCIES: - Stage-8 is a root stage - Stage-1 depends on stages: Stage-8 - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-10 is a root stage + Stage-1 depends on stages: Stage-10 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-8 + Stage: Stage-10 Map Reduce Local Work Alias -> Map Local Tables: x @@ -660,7 +734,7 @@ name: default.srcpart name: default.srcpart - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -779,7 +853,92 @@ name: default.dest_j1 name: default.dest_j1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + numFiles 1 + numPartitions 0 + numRows 107 + rawDataSize 2018 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2125 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + numFiles 1 + numPartitions 0 + numRows 107 + rawDataSize 2018 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2125 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + numFiles 1 + numPartitions 0 + numRows 107 + rawDataSize 2018 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2125 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + name: default.dest_j1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x,y) */ x.key, z.value, y.value FROM src1_copy x JOIN src_copy y ON (x.key = y.key) Index: ql/src/test/results/clientpositive/stats0.q.out =================================================================== --- ql/src/test/results/clientpositive/stats0.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/stats0.q.out (working copy) @@ -1329,11 +1329,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -1411,7 +1413,7 @@ name: default.src name: default.src - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -1510,7 +1512,77 @@ name: default.stats_non_partitioned name: default.stats_non_partitioned + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string +#### A masked pattern was here #### + name default.stats_non_partitioned + serialization.ddl struct stats_non_partitioned { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.stats_non_partitioned + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string +#### A masked pattern was here #### + name default.stats_non_partitioned + serialization.ddl struct stats_non_partitioned { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string +#### A masked pattern was here #### + name default.stats_non_partitioned + serialization.ddl struct stats_non_partitioned { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.stats_non_partitioned + name: default.stats_non_partitioned + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table stats_non_partitioned select * from src PREHOOK: type: QUERY @@ -2085,11 +2157,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -2114,7 +2188,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.stats_partitioned - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -2151,7 +2225,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.stats_partitioned + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.stats_partitioned + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table stats_partitioned partition (ds='1') select * from src PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/input_dynamicserde.q.out =================================================================== --- ql/src/test/results/clientpositive/input_dynamicserde.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input_dynamicserde.q.out (working copy) @@ -28,11 +28,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -63,7 +65,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -98,7 +100,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM src_thrift INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/sample6.q.out =================================================================== --- ql/src/test/results/clientpositive/sample6.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/sample6.q.out (working copy) @@ -18,11 +18,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -107,7 +109,7 @@ name: default.srcbucket name: default.srcbucket - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -206,7 +208,77 @@ name: default.dest1 name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 4 on key) s PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/udf_length.q.out =================================================================== --- ql/src/test/results/clientpositive/udf_length.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/udf_length.q.out (working copy) @@ -25,11 +25,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -52,7 +54,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -87,7 +89,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM src1 INSERT OVERWRITE TABLE dest1 SELECT length(src1.value) PREHOOK: type: QUERY PREHOOK: Input: default@src1 Index: ql/src/test/results/clientpositive/sample1.q.out =================================================================== --- ql/src/test/results/clientpositive/sample1.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/sample1.q.out (working copy) @@ -20,11 +20,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -127,7 +129,7 @@ name: default.srcpart name: default.srcpart - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -226,7 +228,77 @@ name: default.dest1 name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,dt,hr + columns.types int:string:string:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value, string dt, string hr} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,dt,hr + columns.types int:string:string:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value, string dt, string hr} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,dt,hr + columns.types int:string:string:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value, string dt, string hr} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* FROM srcpart TABLESAMPLE (BUCKET 1 OUT OF 1 ON rand()) s WHERE s.ds='2008-04-08' and s.hr='11' Index: ql/src/test/results/clientpositive/input13.q.out =================================================================== --- ql/src/test/results/clientpositive/input13.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input13.q.out (working copy) @@ -32,25 +32,33 @@ STAGE DEPENDENCIES: Stage-4 is a root stage - Stage-8 depends on stages: Stage-4 , consists of Stage-7, Stage-6 + Stage-10 depends on stages: Stage-4 , consists of Stage-7, Stage-6, Stage-8 Stage-7 - Stage-0 depends on stages: Stage-7, Stage-6 + Stage-0 depends on stages: Stage-7, Stage-6, Stage-9 Stage-5 depends on stages: Stage-0 Stage-6 - Stage-12 depends on stages: Stage-4 , consists of Stage-11, Stage-10 - Stage-11 - Stage-1 depends on stages: Stage-11, Stage-10 - Stage-9 depends on stages: Stage-1 - Stage-10 - Stage-16 depends on stages: Stage-4 , consists of Stage-15, Stage-14 - Stage-15 - Stage-2 depends on stages: Stage-15, Stage-14 - Stage-13 depends on stages: Stage-2 + Stage-8 + Stage-9 depends on stages: Stage-8 + Stage-16 depends on stages: Stage-4 , consists of Stage-13, Stage-12, Stage-14 + Stage-13 + Stage-1 depends on stages: Stage-13, Stage-12, Stage-15 + Stage-11 depends on stages: Stage-1 + Stage-12 Stage-14 - Stage-19 depends on stages: Stage-4 , consists of Stage-18, Stage-17 + Stage-15 depends on stages: Stage-14 + Stage-22 depends on stages: Stage-4 , consists of Stage-19, Stage-18, Stage-20 + Stage-19 + Stage-2 depends on stages: Stage-19, Stage-18, Stage-21 + Stage-17 depends on stages: Stage-2 Stage-18 - Stage-3 depends on stages: Stage-18, Stage-17 - Stage-17 + Stage-20 + Stage-21 depends on stages: Stage-20 + Stage-27 depends on stages: Stage-4 , consists of Stage-24, Stage-23, Stage-25 + Stage-24 + Stage-3 depends on stages: Stage-24, Stage-23, Stage-26 + Stage-23 + Stage-25 + Stage-26 depends on stages: Stage-25 STAGE PLANS: Stage: Stage-4 @@ -149,7 +157,7 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - Stage: Stage-8 + Stage: Stage-10 Conditional Operator Stage: Stage-7 @@ -184,10 +192,29 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-12 + Stage: Stage-8 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + + Stage: Stage-9 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-16 Conditional Operator - Stage: Stage-11 + Stage: Stage-13 Move Operator files: hdfs directory: true @@ -203,10 +230,10 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Stage: Stage-9 + Stage: Stage-11 Stats-Aggr Operator - Stage: Stage-10 + Stage: Stage-12 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -219,8 +246,18 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Stage: Stage-16 - Conditional Operator + Stage: Stage-14 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2 Stage: Stage-15 Move Operator @@ -228,6 +265,15 @@ hdfs directory: true #### A masked pattern was here #### + Stage: Stage-22 + Conditional Operator + + Stage: Stage-19 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + Stage: Stage-2 Move Operator tables: @@ -241,10 +287,10 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest3 - Stage: Stage-13 + Stage: Stage-17 Stats-Aggr Operator - Stage: Stage-14 + Stage: Stage-18 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -257,10 +303,29 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest3 - Stage: Stage-19 + Stage: Stage-20 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest3 + + Stage: Stage-21 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-27 Conditional Operator - Stage: Stage-18 + Stage: Stage-24 Move Operator files: hdfs directory: true @@ -272,7 +337,7 @@ hdfs directory: true destination: ../build/ql/test/data/warehouse/dest4.out - Stage: Stage-17 + Stage: Stage-23 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -283,7 +348,24 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + Stage: Stage-25 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + Stage: Stage-26 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.* WHERE src.key < 100 INSERT OVERWRITE TABLE dest2 SELECT src.key, src.value WHERE src.key >= 100 and src.key < 200 Index: ql/src/test/results/clientpositive/input36.q.out =================================================================== --- ql/src/test/results/clientpositive/input36.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input36.q.out (working copy) @@ -26,11 +26,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -74,7 +76,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -109,7 +111,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM ( FROM src SELECT TRANSFORM(src.key, src.value) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002' Index: ql/src/test/results/clientpositive/merge3.q.out =================================================================== --- ql/src/test/results/clientpositive/merge3.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/merge3.q.out (working copy) @@ -54,12 +54,14 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 - Stage-6 depends on stages: Stage-0 - Stage-2 depends on stages: Stage-6 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-8 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-8 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -141,7 +143,7 @@ name: default.merge_src name: default.merge_src - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -156,7 +158,7 @@ hdfs directory: true #### A masked pattern was here #### - Stage: Stage-6 + Stage: Stage-8 Create Table Operator: Create Table columns: key string, value string @@ -217,7 +219,59 @@ name: default.merge_src2 name: default.merge_src2 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types string:string + name default.merge_src2 + serialization.format 1 + name: default.merge_src2 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types string:string + name default.merge_src2 + serialization.format 1 + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types string:string + name default.merge_src2 + serialization.format 1 + name: default.merge_src2 + name: default.merge_src2 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: create table merge_src2 as select key, value from merge_src PREHOOK: type: CREATETABLE_AS_SELECT @@ -2309,11 +2363,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -2454,7 +2510,7 @@ name: default.merge_src_part name: default.merge_src_part - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -2559,7 +2615,80 @@ name: default.merge_src_part2 name: default.merge_src_part2 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string +#### A masked pattern was here #### + name default.merge_src_part2 + partition_columns ds + serialization.ddl struct merge_src_part2 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_src_part2 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string +#### A masked pattern was here #### + name default.merge_src_part2 + partition_columns ds + serialization.ddl struct merge_src_part2 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string +#### A masked pattern was here #### + name default.merge_src_part2 + partition_columns ds + serialization.ddl struct merge_src_part2 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_src_part2 + name: default.merge_src_part2 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table merge_src_part2 partition(ds) select key, value, ds from merge_src_part where ds is not null @@ -4667,11 +4796,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -4836,7 +4967,7 @@ GatherStats: true MultiFileSpray: false - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -4941,7 +5072,80 @@ name: default.merge_src_part2 name: default.merge_src_part2 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string +#### A masked pattern was here #### + name default.merge_src_part2 + partition_columns ds + serialization.ddl struct merge_src_part2 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_src_part2 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string +#### A masked pattern was here #### + name default.merge_src_part2 + partition_columns ds + serialization.ddl struct merge_src_part2 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string +#### A masked pattern was here #### + name default.merge_src_part2 + partition_columns ds + serialization.ddl struct merge_src_part2 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_src_part2 + name: default.merge_src_part2 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: from (select * from merge_src_part where ds is not null distribute by ds) s insert overwrite table merge_src_part2 partition(ds) select key, value, ds Index: ql/src/test/results/clientpositive/union18.q.out =================================================================== --- ql/src/test/results/clientpositive/union18.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/union18.q.out (working copy) @@ -32,16 +32,20 @@ STAGE DEPENDENCIES: Stage-2 is a root stage Stage-3 depends on stages: Stage-2 - Stage-7 depends on stages: Stage-3 , consists of Stage-6, Stage-5 + Stage-9 depends on stages: Stage-3 , consists of Stage-6, Stage-5, Stage-7 Stage-6 - Stage-0 depends on stages: Stage-6, Stage-5 + Stage-0 depends on stages: Stage-6, Stage-5, Stage-8 Stage-4 depends on stages: Stage-0 Stage-5 - Stage-11 depends on stages: Stage-3 , consists of Stage-10, Stage-9 - Stage-10 - Stage-1 depends on stages: Stage-10, Stage-9 - Stage-8 depends on stages: Stage-1 - Stage-9 + Stage-7 + Stage-8 depends on stages: Stage-7 + Stage-15 depends on stages: Stage-3 , consists of Stage-12, Stage-11, Stage-13 + Stage-12 + Stage-1 depends on stages: Stage-12, Stage-11, Stage-14 + Stage-10 depends on stages: Stage-1 + Stage-11 + Stage-13 + Stage-14 depends on stages: Stage-13 STAGE PLANS: Stage: Stage-2 @@ -166,7 +170,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Stage: Stage-7 + Stage: Stage-9 Conditional Operator Stage: Stage-6 @@ -201,10 +205,29 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-11 + Stage: Stage-7 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + + Stage: Stage-8 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-15 Conditional Operator - Stage: Stage-10 + Stage: Stage-12 Move Operator files: hdfs directory: true @@ -220,10 +243,10 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Stage: Stage-8 + Stage: Stage-10 Stats-Aggr Operator - Stage: Stage-9 + Stage: Stage-11 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -236,7 +259,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 + Stage: Stage-13 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2 + Stage: Stage-14 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL select s2.key as key, s2.value as value from src s2) unionsrc Index: ql/src/test/results/clientpositive/multi_insert.q.out =================================================================== --- ql/src/test/results/clientpositive/multi_insert.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/multi_insert.q.out (working copy) @@ -181,16 +181,20 @@ STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 - Stage-10 depends on stages: Stage-2 , consists of Stage-9, Stage-8 - Stage-9 - Stage-1 depends on stages: Stage-9, Stage-8 - Stage-7 depends on stages: Stage-1 - Stage-8 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-14 depends on stages: Stage-2 , consists of Stage-11, Stage-10, Stage-12 + Stage-11 + Stage-1 depends on stages: Stage-11, Stage-10, Stage-13 + Stage-9 depends on stages: Stage-1 + Stage-10 + Stage-12 + Stage-13 depends on stages: Stage-12 STAGE PLANS: Stage: Stage-2 @@ -238,7 +242,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -273,10 +277,29 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi1 - Stage: Stage-10 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-14 Conditional Operator - Stage: Stage-9 + Stage: Stage-11 Move Operator files: hdfs directory: true @@ -292,10 +315,10 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-7 + Stage: Stage-9 Stats-Aggr Operator - Stage: Stage-8 + Stage: Stage-10 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -308,7 +331,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 + Stage: Stage-12 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + Stage: Stage-13 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 insert overwrite table src_multi2 select * where key > 10 and key < 20 @@ -595,16 +637,20 @@ STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 - Stage-10 depends on stages: Stage-2 , consists of Stage-9, Stage-8 - Stage-9 - Stage-1 depends on stages: Stage-9, Stage-8 - Stage-7 depends on stages: Stage-1 - Stage-8 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-14 depends on stages: Stage-2 , consists of Stage-11, Stage-10, Stage-12 + Stage-11 + Stage-1 depends on stages: Stage-11, Stage-10, Stage-13 + Stage-9 depends on stages: Stage-1 + Stage-10 + Stage-12 + Stage-13 depends on stages: Stage-12 STAGE PLANS: Stage: Stage-2 @@ -652,7 +698,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -687,10 +733,29 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi1 - Stage: Stage-10 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-14 Conditional Operator - Stage: Stage-9 + Stage: Stage-11 Move Operator files: hdfs directory: true @@ -706,10 +771,10 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-7 + Stage: Stage-9 Stats-Aggr Operator - Stage: Stage-8 + Stage: Stage-10 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -722,7 +787,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 + Stage: Stage-12 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + Stage: Stage-13 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 insert overwrite table src_multi2 select * where key > 10 and key < 20 @@ -1098,16 +1182,20 @@ STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 - Stage-10 depends on stages: Stage-2 , consists of Stage-9, Stage-8 - Stage-9 - Stage-1 depends on stages: Stage-9, Stage-8 - Stage-7 depends on stages: Stage-1 - Stage-8 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-14 depends on stages: Stage-2 , consists of Stage-11, Stage-10, Stage-12 + Stage-11 + Stage-1 depends on stages: Stage-11, Stage-10, Stage-13 + Stage-9 depends on stages: Stage-1 + Stage-10 + Stage-12 + Stage-13 depends on stages: Stage-12 STAGE PLANS: Stage: Stage-2 @@ -1191,7 +1279,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -1226,10 +1314,29 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi1 - Stage: Stage-10 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-14 Conditional Operator - Stage: Stage-9 + Stage: Stage-11 Move Operator files: hdfs directory: true @@ -1245,10 +1352,10 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-7 + Stage: Stage-9 Stats-Aggr Operator - Stage: Stage-8 + Stage: Stage-10 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -1261,7 +1368,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 + Stage: Stage-12 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + Stage: Stage-13 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value insert overwrite table src_multi2 select * where key > 10 and key < 20 group by key, value @@ -1642,16 +1768,20 @@ STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 - Stage-10 depends on stages: Stage-2 , consists of Stage-9, Stage-8 - Stage-9 - Stage-1 depends on stages: Stage-9, Stage-8 - Stage-7 depends on stages: Stage-1 - Stage-8 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-14 depends on stages: Stage-2 , consists of Stage-11, Stage-10, Stage-12 + Stage-11 + Stage-1 depends on stages: Stage-11, Stage-10, Stage-13 + Stage-9 depends on stages: Stage-1 + Stage-10 + Stage-12 + Stage-13 depends on stages: Stage-12 STAGE PLANS: Stage: Stage-2 @@ -1735,7 +1865,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -1770,10 +1900,29 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi1 - Stage: Stage-10 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-14 Conditional Operator - Stage: Stage-9 + Stage: Stage-11 Move Operator files: hdfs directory: true @@ -1789,10 +1938,10 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-7 + Stage: Stage-9 Stats-Aggr Operator - Stage: Stage-8 + Stage: Stage-10 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -1805,7 +1954,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 + Stage: Stage-12 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + Stage: Stage-13 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value insert overwrite table src_multi2 select * where key > 10 and key < 20 group by key, value @@ -2237,16 +2405,20 @@ STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 - Stage-10 depends on stages: Stage-2 , consists of Stage-9, Stage-8 - Stage-9 - Stage-1 depends on stages: Stage-9, Stage-8 - Stage-7 depends on stages: Stage-1 - Stage-8 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-14 depends on stages: Stage-2 , consists of Stage-11, Stage-10, Stage-12 + Stage-11 + Stage-1 depends on stages: Stage-11, Stage-10, Stage-13 + Stage-9 depends on stages: Stage-1 + Stage-10 + Stage-12 + Stage-13 depends on stages: Stage-12 STAGE PLANS: Stage: Stage-2 @@ -2351,7 +2523,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -2386,10 +2558,29 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi1 - Stage: Stage-10 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-14 Conditional Operator - Stage: Stage-9 + Stage: Stage-11 Move Operator files: hdfs directory: true @@ -2405,10 +2596,10 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-7 + Stage: Stage-9 Stats-Aggr Operator - Stage: Stage-8 + Stage: Stage-10 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -2421,7 +2612,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 + Stage: Stage-12 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + Stage: Stage-13 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 insert overwrite table src_multi2 select * where key > 10 and key < 20 @@ -2931,16 +3141,20 @@ STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 - Stage-10 depends on stages: Stage-2 , consists of Stage-9, Stage-8 - Stage-9 - Stage-1 depends on stages: Stage-9, Stage-8 - Stage-7 depends on stages: Stage-1 - Stage-8 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-14 depends on stages: Stage-2 , consists of Stage-11, Stage-10, Stage-12 + Stage-11 + Stage-1 depends on stages: Stage-11, Stage-10, Stage-13 + Stage-9 depends on stages: Stage-1 + Stage-10 + Stage-12 + Stage-13 depends on stages: Stage-12 STAGE PLANS: Stage: Stage-2 @@ -3045,7 +3259,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -3080,10 +3294,29 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi1 - Stage: Stage-10 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-14 Conditional Operator - Stage: Stage-9 + Stage: Stage-11 Move Operator files: hdfs directory: true @@ -3099,10 +3332,10 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-7 + Stage: Stage-9 Stats-Aggr Operator - Stage: Stage-8 + Stage: Stage-10 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -3115,7 +3348,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 + Stage: Stage-12 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + Stage: Stage-13 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 insert overwrite table src_multi2 select * where key > 10 and key < 20 Index: ql/src/test/results/clientpositive/join37.q.out =================================================================== --- ql/src/test/results/clientpositive/join37.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/join37.q.out (working copy) @@ -17,16 +17,18 @@ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src1) x) (TOK_TABREF (TOK_TABNAME src) y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL y) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest_j1))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST X))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL y) value))))) STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-1 depends on stages: Stage-7 - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-9 Map Reduce Local Work Alias -> Map Local Tables: x @@ -102,7 +104,7 @@ Local Work: Map Reduce Local Work - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -137,7 +139,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_j1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(X) */ x.key, x.value, y.value FROM src1 x JOIN src y ON (x.key = y.key) Index: ql/src/test/results/clientpositive/bucketmapjoin4.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin4.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/bucketmapjoin4.q.out (working copy) @@ -94,16 +94,18 @@ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME srcbucket_mapjoin) a) (TOK_TABREF (TOK_TABNAME srcbucket_mapjoin) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME bucketmapjoin_tmp_result))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST b))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) value))))) STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-1 depends on stages: Stage-7 - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-9 Map Reduce Local Work Alias -> Map Local Tables: b @@ -233,7 +235,7 @@ name: default.srcbucket_mapjoin name: default.srcbucket_mapjoin - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -332,7 +334,77 @@ name: default.bucketmapjoin_tmp_result name: default.bucketmapjoin_tmp_result + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + name: default.bucketmapjoin_tmp_result + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result select /*+mapjoin(b)*/ a.key, a.value, b.value from srcbucket_mapjoin a join srcbucket_mapjoin b @@ -496,16 +568,18 @@ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME srcbucket_mapjoin) a) (TOK_TABREF (TOK_TABNAME srcbucket_mapjoin) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME bucketmapjoin_tmp_result))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST a))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) value))))) STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-1 depends on stages: Stage-7 - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-9 Map Reduce Local Work Alias -> Map Local Tables: a @@ -640,7 +714,7 @@ name: default.srcbucket_mapjoin name: default.srcbucket_mapjoin - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -759,7 +833,92 @@ name: default.bucketmapjoin_tmp_result name: default.bucketmapjoin_tmp_result + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numPartitions 0 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numPartitions 0 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numPartitions 0 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + name: default.bucketmapjoin_tmp_result + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result select /*+mapjoin(a)*/ a.key, a.value, b.value from srcbucket_mapjoin a join srcbucket_mapjoin b Index: ql/src/test/results/clientpositive/index_auto_multiple.q.out =================================================================== --- ql/src/test/results/clientpositive/index_auto_multiple.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/index_auto_multiple.q.out (working copy) @@ -52,11 +52,13 @@ STAGE DEPENDENCIES: Stage-3 is a root stage - Stage-6 depends on stages: Stage-3 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-3 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-2 depends on stages: Stage-5, Stage-4 + Stage-2 depends on stages: Stage-5, Stage-4, Stage-7 Stage-1 depends on stages: Stage-2 Stage-4 + Stage-6 + Stage-7 depends on stages: Stage-6 Stage-0 is a root stage STAGE PLANS: @@ -87,7 +89,7 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -153,6 +155,23 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + Stage: Stage-0 Fetch Operator limit: -1 Index: ql/src/test/results/clientpositive/merge_dynamic_partition2.q.out =================================================================== --- ql/src/test/results/clientpositive/merge_dynamic_partition2.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/merge_dynamic_partition2.q.out (working copy) @@ -57,11 +57,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -88,7 +90,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_dynamic_part - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -126,7 +128,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_dynamic_part + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_dynamic_part + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table merge_dynamic_part partition (ds='2008-04-08', hr) select key, value, hr from srcpart_merge_dp where ds='2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart_merge_dp@ds=2008-04-08/hr=11 Index: ql/src/test/results/clientpositive/union.q.out =================================================================== --- ql/src/test/results/clientpositive/union.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/union.q.out (working copy) @@ -23,10 +23,12 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-4 depends on stages: Stage-1 , consists of Stage-3, Stage-2 + Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2 + Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 Stage-2 + Stage-4 + Stage-5 depends on stages: Stage-4 STAGE PLANS: Stage: Stage-1 @@ -89,7 +91,7 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - Stage: Stage-4 + Stage: Stage-6 Conditional Operator Stage: Stage-3 @@ -115,7 +117,24 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + Stage: Stage-4 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + Stage: Stage-5 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM ( FROM src select src.key, src.value WHERE src.key < 100 UNION ALL Index: ql/src/test/results/clientpositive/udf_10_trims.q.out =================================================================== --- ql/src/test/results/clientpositive/udf_10_trims.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/udf_10_trims.q.out (working copy) @@ -20,11 +20,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -51,7 +53,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -86,7 +88,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT trim(trim(trim(trim(trim(trim(trim(trim(trim(trim( ' abc ')))))))))) FROM src Index: ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out =================================================================== --- ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out (working copy) @@ -185,17 +185,21 @@ STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-7 depends on stages: Stage-5, Stage-4, Stage-10, Stage-9 - Stage-0 depends on stages: Stage-7 + Stage-9 depends on stages: Stage-5, Stage-4, Stage-7, Stage-12, Stage-11, Stage-14 + Stage-0 depends on stages: Stage-9 Stage-3 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-7 - Stage-8 depends on stages: Stage-1 + Stage-1 depends on stages: Stage-9 + Stage-10 depends on stages: Stage-1 Stage-4 - Stage-11 depends on stages: Stage-2 , consists of Stage-10, Stage-9 - Stage-10 - Stage-9 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-15 depends on stages: Stage-2 , consists of Stage-12, Stage-11, Stage-13 + Stage-12 + Stage-11 + Stage-13 + Stage-14 depends on stages: Stage-13 STAGE PLANS: Stage: Stage-2 @@ -243,7 +247,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -252,7 +256,7 @@ hdfs directory: true #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-9 Dependency Collection Stage: Stage-0 @@ -278,7 +282,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-8 + Stage: Stage-10 Stats-Aggr Operator Stage: Stage-4 @@ -294,16 +298,35 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi1 - Stage: Stage-11 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-15 Conditional Operator - Stage: Stage-10 + Stage: Stage-12 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-9 + Stage: Stage-11 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -316,7 +339,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 + Stage: Stage-13 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + Stage: Stage-14 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 insert overwrite table src_multi2 select * where key > 10 and key < 20 @@ -607,17 +649,21 @@ STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-7 depends on stages: Stage-5, Stage-4, Stage-10, Stage-9 - Stage-0 depends on stages: Stage-7 + Stage-9 depends on stages: Stage-5, Stage-4, Stage-7, Stage-12, Stage-11, Stage-14 + Stage-0 depends on stages: Stage-9 Stage-3 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-7 - Stage-8 depends on stages: Stage-1 + Stage-1 depends on stages: Stage-9 + Stage-10 depends on stages: Stage-1 Stage-4 - Stage-11 depends on stages: Stage-2 , consists of Stage-10, Stage-9 - Stage-10 - Stage-9 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-15 depends on stages: Stage-2 , consists of Stage-12, Stage-11, Stage-13 + Stage-12 + Stage-11 + Stage-13 + Stage-14 depends on stages: Stage-13 STAGE PLANS: Stage: Stage-2 @@ -665,7 +711,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -674,7 +720,7 @@ hdfs directory: true #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-9 Dependency Collection Stage: Stage-0 @@ -700,7 +746,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-8 + Stage: Stage-10 Stats-Aggr Operator Stage: Stage-4 @@ -716,16 +762,35 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi1 - Stage: Stage-11 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-15 Conditional Operator - Stage: Stage-10 + Stage: Stage-12 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-9 + Stage: Stage-11 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -738,7 +803,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 + Stage: Stage-13 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + Stage: Stage-14 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 insert overwrite table src_multi2 select * where key > 10 and key < 20 @@ -1118,17 +1202,21 @@ STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-7 depends on stages: Stage-5, Stage-4, Stage-10, Stage-9 - Stage-0 depends on stages: Stage-7 + Stage-9 depends on stages: Stage-5, Stage-4, Stage-7, Stage-12, Stage-11, Stage-14 + Stage-0 depends on stages: Stage-9 Stage-3 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-7 - Stage-8 depends on stages: Stage-1 + Stage-1 depends on stages: Stage-9 + Stage-10 depends on stages: Stage-1 Stage-4 - Stage-11 depends on stages: Stage-2 , consists of Stage-10, Stage-9 - Stage-10 - Stage-9 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-15 depends on stages: Stage-2 , consists of Stage-12, Stage-11, Stage-13 + Stage-12 + Stage-11 + Stage-13 + Stage-14 depends on stages: Stage-13 STAGE PLANS: Stage: Stage-2 @@ -1212,7 +1300,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -1221,7 +1309,7 @@ hdfs directory: true #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-9 Dependency Collection Stage: Stage-0 @@ -1247,7 +1335,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-8 + Stage: Stage-10 Stats-Aggr Operator Stage: Stage-4 @@ -1263,16 +1351,35 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi1 - Stage: Stage-11 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-15 Conditional Operator - Stage: Stage-10 + Stage: Stage-12 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-9 + Stage: Stage-11 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -1285,7 +1392,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 + Stage: Stage-13 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + Stage: Stage-14 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value insert overwrite table src_multi2 select * where key > 10 and key < 20 group by key, value @@ -1670,17 +1796,21 @@ STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-7 depends on stages: Stage-5, Stage-4, Stage-10, Stage-9 - Stage-0 depends on stages: Stage-7 + Stage-9 depends on stages: Stage-5, Stage-4, Stage-7, Stage-12, Stage-11, Stage-14 + Stage-0 depends on stages: Stage-9 Stage-3 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-7 - Stage-8 depends on stages: Stage-1 + Stage-1 depends on stages: Stage-9 + Stage-10 depends on stages: Stage-1 Stage-4 - Stage-11 depends on stages: Stage-2 , consists of Stage-10, Stage-9 - Stage-10 - Stage-9 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-15 depends on stages: Stage-2 , consists of Stage-12, Stage-11, Stage-13 + Stage-12 + Stage-11 + Stage-13 + Stage-14 depends on stages: Stage-13 STAGE PLANS: Stage: Stage-2 @@ -1764,7 +1894,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -1773,7 +1903,7 @@ hdfs directory: true #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-9 Dependency Collection Stage: Stage-0 @@ -1799,7 +1929,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-8 + Stage: Stage-10 Stats-Aggr Operator Stage: Stage-4 @@ -1815,16 +1945,35 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi1 - Stage: Stage-11 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-15 Conditional Operator - Stage: Stage-10 + Stage: Stage-12 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-9 + Stage: Stage-11 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -1837,7 +1986,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 + Stage: Stage-13 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + Stage: Stage-14 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value insert overwrite table src_multi2 select * where key > 10 and key < 20 group by key, value @@ -2273,17 +2441,21 @@ STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-7 depends on stages: Stage-5, Stage-4, Stage-10, Stage-9 - Stage-0 depends on stages: Stage-7 + Stage-9 depends on stages: Stage-5, Stage-4, Stage-7, Stage-12, Stage-11, Stage-14 + Stage-0 depends on stages: Stage-9 Stage-3 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-7 - Stage-8 depends on stages: Stage-1 + Stage-1 depends on stages: Stage-9 + Stage-10 depends on stages: Stage-1 Stage-4 - Stage-11 depends on stages: Stage-2 , consists of Stage-10, Stage-9 - Stage-10 - Stage-9 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-15 depends on stages: Stage-2 , consists of Stage-12, Stage-11, Stage-13 + Stage-12 + Stage-11 + Stage-13 + Stage-14 depends on stages: Stage-13 STAGE PLANS: Stage: Stage-2 @@ -2388,7 +2560,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -2397,7 +2569,7 @@ hdfs directory: true #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-9 Dependency Collection Stage: Stage-0 @@ -2423,7 +2595,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-8 + Stage: Stage-10 Stats-Aggr Operator Stage: Stage-4 @@ -2439,16 +2611,35 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi1 - Stage: Stage-11 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-15 Conditional Operator - Stage: Stage-10 + Stage: Stage-12 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-9 + Stage: Stage-11 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -2461,7 +2652,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 + Stage: Stage-13 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + Stage: Stage-14 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 insert overwrite table src_multi2 select * where key > 10 and key < 20 @@ -2975,17 +3185,21 @@ STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-7 depends on stages: Stage-5, Stage-4, Stage-10, Stage-9 - Stage-0 depends on stages: Stage-7 + Stage-9 depends on stages: Stage-5, Stage-4, Stage-7, Stage-12, Stage-11, Stage-14 + Stage-0 depends on stages: Stage-9 Stage-3 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-7 - Stage-8 depends on stages: Stage-1 + Stage-1 depends on stages: Stage-9 + Stage-10 depends on stages: Stage-1 Stage-4 - Stage-11 depends on stages: Stage-2 , consists of Stage-10, Stage-9 - Stage-10 - Stage-9 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-15 depends on stages: Stage-2 , consists of Stage-12, Stage-11, Stage-13 + Stage-12 + Stage-11 + Stage-13 + Stage-14 depends on stages: Stage-13 STAGE PLANS: Stage: Stage-2 @@ -3090,7 +3304,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -3099,7 +3313,7 @@ hdfs directory: true #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-9 Dependency Collection Stage: Stage-0 @@ -3125,7 +3339,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-8 + Stage: Stage-10 Stats-Aggr Operator Stage: Stage-4 @@ -3141,16 +3355,35 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi1 - Stage: Stage-11 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-15 Conditional Operator - Stage: Stage-10 + Stage: Stage-12 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-9 + Stage: Stage-11 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -3163,7 +3396,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 + Stage: Stage-13 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + Stage: Stage-14 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 insert overwrite table src_multi2 select * where key > 10 and key < 20 @@ -5958,21 +6210,25 @@ STAGE DEPENDENCIES: Stage-4 is a root stage - Stage-8 depends on stages: Stage-4 , consists of Stage-7, Stage-6 + Stage-10 depends on stages: Stage-4 , consists of Stage-7, Stage-6, Stage-8 Stage-7 - Stage-9 depends on stages: Stage-7, Stage-6, Stage-12, Stage-11, Stage-14, Stage-15 - Stage-0 depends on stages: Stage-9 + Stage-11 depends on stages: Stage-7, Stage-6, Stage-9, Stage-14, Stage-13, Stage-16, Stage-18, Stage-19 + Stage-0 depends on stages: Stage-11 Stage-5 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-9 - Stage-10 depends on stages: Stage-1 + Stage-1 depends on stages: Stage-11 + Stage-12 depends on stages: Stage-1 Stage-6 - Stage-13 depends on stages: Stage-4 , consists of Stage-12, Stage-11 - Stage-12 - Stage-11 - Stage-14 depends on stages: Stage-4 - Stage-2 depends on stages: Stage-14 - Stage-15 depends on stages: Stage-4 - Stage-3 depends on stages: Stage-15 + Stage-8 + Stage-9 depends on stages: Stage-8 + Stage-17 depends on stages: Stage-4 , consists of Stage-14, Stage-13, Stage-15 + Stage-14 + Stage-13 + Stage-15 + Stage-16 depends on stages: Stage-15 + Stage-18 depends on stages: Stage-4 + Stage-2 depends on stages: Stage-18 + Stage-19 depends on stages: Stage-4 + Stage-3 depends on stages: Stage-19 STAGE PLANS: Stage: Stage-4 @@ -6098,7 +6354,7 @@ input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - Stage: Stage-8 + Stage: Stage-10 Conditional Operator Stage: Stage-7 @@ -6107,7 +6363,7 @@ hdfs directory: true #### A masked pattern was here #### - Stage: Stage-9 + Stage: Stage-11 Dependency Collection Stage: Stage-0 @@ -6133,7 +6389,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-10 + Stage: Stage-12 Stats-Aggr Operator Stage: Stage-6 @@ -6149,16 +6405,35 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi1 - Stage: Stage-13 + Stage: Stage-8 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + + Stage: Stage-9 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-17 Conditional Operator - Stage: Stage-12 + Stage: Stage-14 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-11 + Stage: Stage-13 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -6171,10 +6446,29 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-14 + Stage: Stage-15 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + + Stage: Stage-16 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-18 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### Reduce Output Operator key expressions: expr: _col0 @@ -6204,7 +6498,7 @@ hdfs directory: false #### A masked pattern was here #### - Stage: Stage-15 + Stage: Stage-19 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -6486,21 +6780,25 @@ STAGE DEPENDENCIES: Stage-4 is a root stage - Stage-8 depends on stages: Stage-4 , consists of Stage-7, Stage-6 + Stage-10 depends on stages: Stage-4 , consists of Stage-7, Stage-6, Stage-8 Stage-7 - Stage-9 depends on stages: Stage-7, Stage-6, Stage-12, Stage-11, Stage-14, Stage-15 - Stage-0 depends on stages: Stage-9 + Stage-11 depends on stages: Stage-7, Stage-6, Stage-9, Stage-14, Stage-13, Stage-16, Stage-18, Stage-19 + Stage-0 depends on stages: Stage-11 Stage-5 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-9 - Stage-10 depends on stages: Stage-1 + Stage-1 depends on stages: Stage-11 + Stage-12 depends on stages: Stage-1 Stage-6 - Stage-13 depends on stages: Stage-4 , consists of Stage-12, Stage-11 - Stage-12 - Stage-11 - Stage-14 depends on stages: Stage-4 - Stage-2 depends on stages: Stage-14 - Stage-15 depends on stages: Stage-4 - Stage-3 depends on stages: Stage-15 + Stage-8 + Stage-9 depends on stages: Stage-8 + Stage-17 depends on stages: Stage-4 , consists of Stage-14, Stage-13, Stage-15 + Stage-14 + Stage-13 + Stage-15 + Stage-16 depends on stages: Stage-15 + Stage-18 depends on stages: Stage-4 + Stage-2 depends on stages: Stage-18 + Stage-19 depends on stages: Stage-4 + Stage-3 depends on stages: Stage-19 STAGE PLANS: Stage: Stage-4 @@ -6626,7 +6924,7 @@ input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - Stage: Stage-8 + Stage: Stage-10 Conditional Operator Stage: Stage-7 @@ -6635,7 +6933,7 @@ hdfs directory: true #### A masked pattern was here #### - Stage: Stage-9 + Stage: Stage-11 Dependency Collection Stage: Stage-0 @@ -6661,7 +6959,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-10 + Stage: Stage-12 Stats-Aggr Operator Stage: Stage-6 @@ -6677,16 +6975,35 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi1 - Stage: Stage-13 + Stage: Stage-8 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi1 + + Stage: Stage-9 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-17 Conditional Operator - Stage: Stage-12 + Stage: Stage-14 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-11 + Stage: Stage-13 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -6699,10 +7016,29 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Stage: Stage-14 + Stage: Stage-15 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_multi2 + + Stage: Stage-16 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-18 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### Reduce Output Operator key expressions: expr: _col0 @@ -6732,7 +7068,7 @@ hdfs directory: false #### A masked pattern was here #### - Stage: Stage-15 + Stage: Stage-19 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### Index: ql/src/test/results/clientpositive/input_part1.q.out =================================================================== --- ql/src/test/results/clientpositive/input_part1.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input_part1.q.out (working copy) @@ -16,11 +16,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -123,7 +125,7 @@ name: default.srcpart name: default.srcpart - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -222,7 +224,77 @@ name: default.dest1 name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,hr,ds + columns.types int:string:string:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value, string hr, string ds} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,hr,ds + columns.types int:string:string:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value, string hr, string ds} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,hr,ds + columns.types int:string:string:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value, string hr, string ds} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM srcpart INSERT OVERWRITE TABLE dest1 SELECT srcpart.key, srcpart.value, srcpart.hr, srcpart.ds WHERE srcpart.key < 100 and srcpart.ds = '2008-04-08' and srcpart.hr = '12' PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/rcfile_merge1.q.out =================================================================== --- ql/src/test/results/clientpositive/rcfile_merge1.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/rcfile_merge1.q.out (working copy) @@ -37,11 +37,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -77,7 +79,7 @@ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.rcfile_merge1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -115,7 +117,26 @@ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.rcfile_merge1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.rcfile_merge1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE rcfile_merge1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 100) as part FROM src @@ -637,11 +658,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -677,7 +700,7 @@ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.rcfile_merge1b - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -705,7 +728,16 @@ Stage: Stage-3 Block level merge + Stage: Stage-5 + Block level merge + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE rcfile_merge1b PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 100) as part FROM src Index: ql/src/test/results/clientpositive/union4.q.out =================================================================== --- ql/src/test/results/clientpositive/union4.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/union4.q.out (working copy) @@ -26,13 +26,15 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-7 - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-2 depends on stages: Stage-1, Stage-9 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 - Stage-7 is a root stage + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-9 is a root stage STAGE PLANS: Stage: Stage-1 @@ -129,7 +131,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -164,7 +166,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-9 Map Reduce Alias -> Map Operator Tree: null-subquery2:unionsrc-subquery2:s2 Index: ql/src/test/results/clientpositive/insert1.q.out =================================================================== --- ql/src/test/results/clientpositive/insert1.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/insert1.q.out (working copy) @@ -29,11 +29,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -62,7 +64,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.insert1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -97,7 +99,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.insert1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: explain insert into table INSERT1 select a.key, a.value from insert2 a WHERE (a.key=-1) PREHOOK: type: QUERY POSTHOOK: query: explain insert into table INSERT1 select a.key, a.value from insert2 a WHERE (a.key=-1) @@ -109,11 +130,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -142,7 +165,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.insert1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -177,4 +200,23 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.insert1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Index: ql/src/test/results/clientpositive/input38.q.out =================================================================== --- ql/src/test/results/clientpositive/input38.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input38.q.out (working copy) @@ -24,11 +24,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -69,7 +71,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -104,7 +106,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM ( FROM src SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) Index: ql/src/test/results/clientpositive/union29.q.out =================================================================== --- ql/src/test/results/clientpositive/union29.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/union29.q.out (working copy) @@ -34,11 +34,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -160,7 +162,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -195,7 +197,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table union_subq_union select * from ( select key, value from src Index: ql/src/test/results/clientpositive/quote1.q.out =================================================================== --- ql/src/test/results/clientpositive/quote1.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/quote1.q.out (working copy) @@ -16,11 +16,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -56,7 +58,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -93,7 +95,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: EXPLAIN SELECT `int`.`location`, `int`.`type`, `int`.`table` FROM dest1 `int` WHERE `int`.`table` = '2008-04-08' PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/load_dyn_part14.q.out =================================================================== --- ql/src/test/results/clientpositive/load_dyn_part14.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/load_dyn_part14.q.out (working copy) @@ -38,14 +38,16 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-7, Stage-8 - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-2 depends on stages: Stage-1, Stage-9, Stage-10 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 - Stage-7 is a root stage - Stage-8 is a root stage + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-9 is a root stage + Stage-10 is a root stage STAGE PLANS: Stage: Stage-1 @@ -138,7 +140,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part14 - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -175,7 +177,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part14 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part14 + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-9 Map Reduce Alias -> Map Operator Tree: null-subquery2:t-subquery2:src @@ -207,7 +228,7 @@ input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - Stage: Stage-8 + Stage: Stage-10 Map Reduce Alias -> Map Operator Tree: null-subquery1-subquery1:t-subquery1-subquery1:src Index: ql/src/test/results/clientpositive/input7.q.out =================================================================== --- ql/src/test/results/clientpositive/input7.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input7.q.out (working copy) @@ -16,11 +16,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -52,7 +54,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -87,7 +89,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM src1 INSERT OVERWRITE TABLE dest1 SELECT NULL, src1.key PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/join39.q.out =================================================================== --- ql/src/test/results/clientpositive/join39.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/join39.q.out (working copy) @@ -17,16 +17,18 @@ (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_TABREF (TOK_TABNAME src) x) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (<= (TOK_TABLE_OR_COL key) 100)))) y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL y) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest_j1))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST y))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL y) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL y) value))))) STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-1 depends on stages: Stage-7 - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-9 Map Reduce Local Work Alias -> Map Local Tables: y:src @@ -108,7 +110,7 @@ Local Work: Map Reduce Local Work - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -143,7 +145,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_j1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(y) */ x.key, x.value, y.key, y.value FROM src x left outer JOIN (select * from src where key <= 100) y ON (x.key = y.key) Index: ql/src/test/results/clientpositive/insert_into5.q.out =================================================================== --- ql/src/test/results/clientpositive/insert_into5.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/insert_into5.q.out (working copy) @@ -113,11 +113,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -142,7 +144,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.insert_into5a - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -177,7 +179,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.insert_into5a + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into5a + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT INTO TABLE insert_into5a SELECT * FROM insert_into5a PREHOOK: type: QUERY PREHOOK: Input: default@insert_into5a @@ -222,11 +243,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -251,7 +274,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.insert_into5b - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -288,7 +311,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.insert_into5b + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into5b + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT INTO TABLE insert_into5b PARTITION (ds='1') SELECT * FROM insert_into5a PREHOOK: type: QUERY PREHOOK: Input: default@insert_into5a @@ -339,11 +381,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -368,7 +412,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.insert_into5b - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -405,7 +449,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.insert_into5b + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into5b + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT INTO TABLE insert_into5b PARTITION (ds='1') SELECT key, value FROM insert_into5b PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/index_auto.q.out =================================================================== --- ql/src/test/results/clientpositive/index_auto.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/index_auto.q.out (working copy) @@ -161,11 +161,13 @@ STAGE DEPENDENCIES: Stage-3 is a root stage - Stage-6 depends on stages: Stage-3 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-3 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-2 depends on stages: Stage-5, Stage-4 + Stage-2 depends on stages: Stage-5, Stage-4, Stage-7 Stage-1 depends on stages: Stage-2 Stage-4 + Stage-6 + Stage-7 depends on stages: Stage-6 Stage-0 is a root stage STAGE PLANS: @@ -196,7 +198,7 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -262,6 +264,23 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + Stage: Stage-0 Fetch Operator limit: -1 Index: ql/src/test/results/clientpositive/join25.q.out =================================================================== --- ql/src/test/results/clientpositive/join25.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/join25.q.out (working copy) @@ -17,16 +17,18 @@ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src1) x) (TOK_TABREF (TOK_TABNAME src) y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL y) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest_j1))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST x))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL y) value))))) STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-1 depends on stages: Stage-7 - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-9 Map Reduce Local Work Alias -> Map Local Tables: x @@ -102,7 +104,7 @@ Local Work: Map Reduce Local Work - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -137,7 +139,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_j1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, x.value, y.value FROM src1 x JOIN src y ON (x.key = y.key) Index: ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out =================================================================== --- ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out (working copy) @@ -119,11 +119,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -150,7 +152,7 @@ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.merge_dynamic_part - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -178,7 +180,16 @@ Stage: Stage-3 Block level merge + Stage: Stage-5 + Block level merge + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table merge_dynamic_part partition (ds = '2008-04-08', hr) select key, value, if(key % 2 == 0, 'a1', 'b1') as hr from srcpart_merge_dp_rc where ds = '2008-04-08' PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/union10.q.out =================================================================== --- ql/src/test/results/clientpositive/union10.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/union10.q.out (working copy) @@ -28,14 +28,16 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-7, Stage-8 - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-2 depends on stages: Stage-1, Stage-9, Stage-10 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 - Stage-7 is a root stage - Stage-8 is a root stage + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-9 is a root stage + Stage-10 is a root stage STAGE PLANS: Stage: Stage-1 @@ -157,7 +159,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -192,7 +194,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-9 Map Reduce Alias -> Map Operator Tree: null-subquery1-subquery2:unionsrc-subquery1-subquery2:s2 @@ -232,7 +253,7 @@ input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - Stage: Stage-8 + Stage: Stage-10 Map Reduce Alias -> Map Operator Tree: null-subquery2:unionsrc-subquery2:s3 Index: ql/src/test/results/clientpositive/join34.q.out =================================================================== --- ql/src/test/results/clientpositive/join34.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/join34.q.out (working copy) @@ -27,16 +27,18 @@ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src) x)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) value) value)) (TOK_WHERE (< (. (TOK_TABLE_OR_COL x) key) 20)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src) x1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL x1) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x1) value) value)) (TOK_WHERE (> (. (TOK_TABLE_OR_COL x1) key) 100))))) subq1) (TOK_TABREF (TOK_TABNAME src1) x) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL subq1) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest_j1))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST x))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL subq1) value))))) STAGE DEPENDENCIES: - Stage-8 is a root stage - Stage-1 depends on stages: Stage-8 - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-10 is a root stage + Stage-1 depends on stages: Stage-10 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-8 + Stage: Stage-10 Map Reduce Local Work Alias -> Map Local Tables: x @@ -241,7 +243,7 @@ name: default.src name: default.src - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -340,7 +342,77 @@ name: default.dest_j1 name: default.dest_j1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + name: default.dest_j1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, x.value, subq1.value FROM Index: ql/src/test/results/clientpositive/bucketmapjoin1.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin1.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/bucketmapjoin1.q.out (working copy) @@ -346,16 +346,18 @@ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME srcbucket_mapjoin) a) (TOK_TABREF (TOK_TABNAME srcbucket_mapjoin_part) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME bucketmapjoin_tmp_result))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST b))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) value))) (TOK_WHERE (= (. (TOK_TABLE_OR_COL b) ds) "2008-04-08")))) STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-1 depends on stages: Stage-7 - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-9 Map Reduce Local Work Alias -> Map Local Tables: b @@ -487,7 +489,7 @@ name: default.srcbucket_mapjoin name: default.srcbucket_mapjoin - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -586,7 +588,77 @@ name: default.bucketmapjoin_tmp_result name: default.bucketmapjoin_tmp_result + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + name: default.bucketmapjoin_tmp_result + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result select /*+mapjoin(b)*/ a.key, a.value, b.value from srcbucket_mapjoin a join srcbucket_mapjoin_part b @@ -754,16 +826,18 @@ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME srcbucket_mapjoin) a) (TOK_TABREF (TOK_TABNAME srcbucket_mapjoin_part) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME bucketmapjoin_tmp_result))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST a))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) value))) (TOK_WHERE (= (. (TOK_TABLE_OR_COL b) ds) "2008-04-08")))) STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-1 depends on stages: Stage-7 - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-9 Map Reduce Local Work Alias -> Map Local Tables: a @@ -904,7 +978,7 @@ name: default.srcbucket_mapjoin_part name: default.srcbucket_mapjoin_part - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -1023,7 +1097,92 @@ name: default.bucketmapjoin_tmp_result name: default.bucketmapjoin_tmp_result + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numPartitions 0 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numPartitions 0 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numPartitions 0 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + name: default.bucketmapjoin_tmp_result + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result select /*+mapjoin(a)*/ a.key, a.value, b.value from srcbucket_mapjoin a join srcbucket_mapjoin_part b Index: ql/src/test/results/clientpositive/rcfile_merge3.q.out =================================================================== --- ql/src/test/results/clientpositive/rcfile_merge3.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/rcfile_merge3.q.out (working copy) @@ -59,11 +59,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -88,7 +90,7 @@ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.rcfile_merge3b - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -113,7 +115,16 @@ Stage: Stage-3 Block level merge + Stage: Stage-5 + Block level merge + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE rcfile_merge3b SELECT key, value FROM rcfile_merge3a PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/subq.q.out =================================================================== --- ql/src/test/results/clientpositive/subq.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/subq.q.out (working copy) @@ -15,10 +15,12 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-4 depends on stages: Stage-1 , consists of Stage-3, Stage-2 + Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2 + Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 Stage-2 + Stage-4 + Stage-5 depends on stages: Stage-4 STAGE PLANS: Stage: Stage-1 @@ -52,7 +54,7 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - Stage: Stage-4 + Stage: Stage-6 Conditional Operator Stage: Stage-3 @@ -78,7 +80,24 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + Stage: Stage-4 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + Stage: Stage-5 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM ( FROM src select src.* WHERE src.key < 100 ) unioninput Index: ql/src/test/results/clientpositive/union6.q.out =================================================================== --- ql/src/test/results/clientpositive/union6.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/union6.q.out (working copy) @@ -27,11 +27,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 + Stage-6 + Stage-7 depends on stages: Stage-6 STAGE PLANS: Stage: Stage-1 @@ -122,7 +124,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -157,7 +159,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL Index: ql/src/test/results/clientpositive/sample5.q.out =================================================================== --- ql/src/test/results/clientpositive/sample5.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/sample5.q.out (working copy) @@ -18,11 +18,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -107,7 +109,7 @@ name: default.srcbucket name: default.srcbucket - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -206,7 +208,77 @@ name: default.dest1 name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* -- here's another test FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/rand_partitionpruner2.q.out =================================================================== --- ql/src/test/results/clientpositive/rand_partitionpruner2.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/rand_partitionpruner2.q.out (working copy) @@ -20,11 +20,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -153,7 +155,7 @@ name: default.srcpart name: default.srcpart - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -252,7 +254,77 @@ name: default.tmptable name: default.tmptable + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,hr,ds + columns.types string:string:string:string +#### A masked pattern was here #### + name default.tmptable + serialization.ddl struct tmptable { string key, string value, string hr, string ds} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,hr,ds + columns.types string:string:string:string +#### A masked pattern was here #### + name default.tmptable + serialization.ddl struct tmptable { string key, string value, string hr, string ds} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,hr,ds + columns.types string:string:string:string +#### A masked pattern was here #### + name default.tmptable + serialization.ddl struct tmptable { string key, string value, string hr, string ds} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + name: default.tmptable + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table tmptable select a.* from srcpart a where rand(1) < 0.1 and a.ds = '2008-04-08' PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/input12.q.out =================================================================== --- ql/src/test/results/clientpositive/input12.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input12.q.out (working copy) @@ -30,21 +30,27 @@ STAGE DEPENDENCIES: Stage-3 is a root stage - Stage-7 depends on stages: Stage-3 , consists of Stage-6, Stage-5 + Stage-9 depends on stages: Stage-3 , consists of Stage-6, Stage-5, Stage-7 Stage-6 - Stage-0 depends on stages: Stage-6, Stage-5 + Stage-0 depends on stages: Stage-6, Stage-5, Stage-8 Stage-4 depends on stages: Stage-0 Stage-5 - Stage-11 depends on stages: Stage-3 , consists of Stage-10, Stage-9 - Stage-10 - Stage-1 depends on stages: Stage-10, Stage-9 - Stage-8 depends on stages: Stage-1 - Stage-9 - Stage-15 depends on stages: Stage-3 , consists of Stage-14, Stage-13 - Stage-14 - Stage-2 depends on stages: Stage-14, Stage-13 - Stage-12 depends on stages: Stage-2 + Stage-7 + Stage-8 depends on stages: Stage-7 + Stage-15 depends on stages: Stage-3 , consists of Stage-12, Stage-11, Stage-13 + Stage-12 + Stage-1 depends on stages: Stage-12, Stage-11, Stage-14 + Stage-10 depends on stages: Stage-1 + Stage-11 Stage-13 + Stage-14 depends on stages: Stage-13 + Stage-21 depends on stages: Stage-3 , consists of Stage-18, Stage-17, Stage-19 + Stage-18 + Stage-2 depends on stages: Stage-18, Stage-17, Stage-20 + Stage-16 depends on stages: Stage-2 + Stage-17 + Stage-19 + Stage-20 depends on stages: Stage-19 STAGE PLANS: Stage: Stage-3 @@ -128,7 +134,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest3 - Stage: Stage-7 + Stage: Stage-9 Conditional Operator Stage: Stage-6 @@ -163,10 +169,29 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-11 + Stage: Stage-7 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + + Stage: Stage-8 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-15 Conditional Operator - Stage: Stage-10 + Stage: Stage-12 Move Operator files: hdfs directory: true @@ -182,10 +207,10 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Stage: Stage-8 + Stage: Stage-10 Stats-Aggr Operator - Stage: Stage-9 + Stage: Stage-11 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -198,8 +223,18 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Stage: Stage-15 - Conditional Operator + Stage: Stage-13 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2 Stage: Stage-14 Move Operator @@ -207,6 +242,15 @@ hdfs directory: true #### A masked pattern was here #### + Stage: Stage-21 + Conditional Operator + + Stage: Stage-18 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + Stage: Stage-2 Move Operator tables: @@ -220,10 +264,10 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest3 - Stage: Stage-12 + Stage: Stage-16 Stats-Aggr Operator - Stage: Stage-13 + Stage: Stage-17 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -236,7 +280,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest3 + Stage: Stage-19 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest3 + Stage: Stage-20 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.* WHERE src.key < 100 INSERT OVERWRITE TABLE dest2 SELECT src.key, src.value WHERE src.key >= 100 and src.key < 200 Index: ql/src/test/results/clientpositive/input35.q.out =================================================================== --- ql/src/test/results/clientpositive/input35.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input35.q.out (working copy) @@ -26,11 +26,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -74,7 +76,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -109,7 +111,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM ( FROM src SELECT TRANSFORM(src.key, src.value) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002' Index: ql/src/test/results/clientpositive/merge2.q.out =================================================================== --- ql/src/test/results/clientpositive/merge2.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/merge2.q.out (working copy) @@ -16,11 +16,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -88,7 +90,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.test1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -123,7 +125,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.test1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table test1 select key, count(1) from src group by key PREHOOK: type: QUERY @@ -522,11 +543,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -549,7 +572,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.test1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -584,7 +607,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.test1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table test1 select key from test_src PREHOOK: type: QUERY PREHOOK: Input: default@test_src@ds=101 @@ -620,11 +662,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -647,7 +691,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.test1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -682,7 +726,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.test1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table test1 select key from test_src PREHOOK: type: QUERY PREHOOK: Input: default@test_src@ds=101 Index: ql/src/test/results/clientpositive/input9.q.out =================================================================== --- ql/src/test/results/clientpositive/input9.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input9.q.out (working copy) @@ -16,11 +16,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -56,7 +58,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -91,7 +93,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM src1 INSERT OVERWRITE TABLE dest1 SELECT NULL, src1.key where NULL = NULL PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/union12.q.out =================================================================== --- ql/src/test/results/clientpositive/union12.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/union12.q.out (working copy) @@ -28,14 +28,16 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-7, Stage-8 - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-2 depends on stages: Stage-1, Stage-9, Stage-10 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 - Stage-7 is a root stage - Stage-8 is a root stage + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-9 is a root stage + Stage-10 is a root stage STAGE PLANS: Stage: Stage-1 @@ -157,7 +159,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -192,7 +194,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-9 Map Reduce Alias -> Map Operator Tree: null-subquery1-subquery2:unionsrc-subquery1-subquery2:s2 @@ -232,7 +253,7 @@ input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - Stage: Stage-8 + Stage: Stage-10 Map Reduce Alias -> Map Operator Tree: null-subquery2:unionsrc-subquery2:s3 Index: ql/src/test/results/clientpositive/join36.q.out =================================================================== --- ql/src/test/results/clientpositive/join36.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/join36.q.out (working copy) @@ -57,16 +57,18 @@ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME tmp1) x) (TOK_TABREF (TOK_TABNAME tmp2) y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL y) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest_j1))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST x))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) cnt)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL y) cnt))))) STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-1 depends on stages: Stage-7 - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-9 Map Reduce Local Work Alias -> Map Local Tables: x @@ -133,7 +135,7 @@ Local Work: Map Reduce Local Work - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -168,7 +170,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_j1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, x.cnt, y.cnt FROM tmp1 x JOIN tmp2 y ON (x.key = y.key) Index: ql/src/test/results/clientpositive/input_part5.q.out =================================================================== --- ql/src/test/results/clientpositive/input_part5.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input_part5.q.out (working copy) @@ -16,11 +16,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -53,7 +55,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -88,7 +90,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table tmptable SELECT x.* FROM SRCPART x WHERE x.ds = '2008-04-08' and x.key < 100 PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/cast1.q.out =================================================================== --- ql/src/test/results/clientpositive/cast1.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/cast1.q.out (working copy) @@ -14,11 +14,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -57,7 +59,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -92,7 +94,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT 3 + 2, 3.0 + 2, 3 + 2.0, 3.0 + 2.0, 3 + CAST(2.0 AS INT) + CAST(CAST(0 AS SMALLINT) AS INT), CAST(1 AS BOOLEAN), CAST(TRUE AS INT) WHERE src.key = 86 PREHOOK: type: QUERY PREHOOK: Input: default@src Index: ql/src/test/results/clientpositive/ppd_constant_expr.q.out =================================================================== --- ql/src/test/results/clientpositive/ppd_constant_expr.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/ppd_constant_expr.q.out (working copy) @@ -16,11 +16,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -56,7 +58,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.ppd_constant_expr - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -91,7 +93,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.ppd_constant_expr + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.ppd_constant_expr + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM src1 INSERT OVERWRITE TABLE ppd_constant_expr SELECT 4 + NULL, src1.key - NULL, NULL + NULL PREHOOK: type: QUERY @@ -157,11 +178,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -197,7 +220,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.ppd_constant_expr - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -232,7 +255,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.ppd_constant_expr + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.ppd_constant_expr + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM src1 INSERT OVERWRITE TABLE ppd_constant_expr SELECT 4 + NULL, src1.key - NULL, NULL + NULL PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/union30.q.out =================================================================== --- ql/src/test/results/clientpositive/union30.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/union30.q.out (working copy) @@ -48,15 +48,17 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-9 + Stage-2 depends on stages: Stage-1, Stage-11 Stage-3 depends on stages: Stage-2 Stage-4 depends on stages: Stage-3 - Stage-8 depends on stages: Stage-4 , consists of Stage-7, Stage-6 + Stage-10 depends on stages: Stage-4 , consists of Stage-7, Stage-6, Stage-8 Stage-7 - Stage-0 depends on stages: Stage-7, Stage-6 + Stage-0 depends on stages: Stage-7, Stage-6, Stage-9 Stage-5 depends on stages: Stage-0 Stage-6 - Stage-9 is a root stage + Stage-8 + Stage-9 depends on stages: Stage-8 + Stage-11 is a root stage STAGE PLANS: Stage: Stage-1 @@ -269,7 +271,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union - Stage: Stage-8 + Stage: Stage-10 Conditional Operator Stage: Stage-7 @@ -304,7 +306,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union + Stage: Stage-8 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union + Stage: Stage-9 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-11 Map Reduce Alias -> Map Operator Tree: null-subquery1:aa-subquery1-subquery2:a-subquery2-subquery2:subq-subquery2:src Index: ql/src/test/results/clientpositive/udf_reverse.q.out =================================================================== --- ql/src/test/results/clientpositive/udf_reverse.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/udf_reverse.q.out (working copy) @@ -25,11 +25,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -52,7 +54,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -87,7 +89,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM src1 INSERT OVERWRITE TABLE dest1 SELECT reverse(src1.value) PREHOOK: type: QUERY PREHOOK: Input: default@src1 Index: ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out (working copy) @@ -67,16 +67,18 @@ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME srcbucket_mapjoin) a) (TOK_TABREF (TOK_TABNAME srcbucket_mapjoin_part_2) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME bucketmapjoin_tmp_result))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST b))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) value))))) STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-1 depends on stages: Stage-7 - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-9 Map Reduce Local Work Alias -> Map Local Tables: b @@ -206,7 +208,7 @@ name: default.srcbucket_mapjoin name: default.srcbucket_mapjoin - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -305,4 +307,74 @@ name: default.bucketmapjoin_tmp_result name: default.bucketmapjoin_tmp_result + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + name: default.bucketmapjoin_tmp_result + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Index: ql/src/test/results/clientpositive/sample7.q.out =================================================================== --- ql/src/test/results/clientpositive/sample7.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/sample7.q.out (working copy) @@ -20,11 +20,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -114,7 +116,7 @@ name: default.srcbucket name: default.srcbucket - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -213,7 +215,77 @@ name: default.dest1 name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 4 on key) s WHERE s.key > 100 Index: ql/src/test/results/clientpositive/input_testsequencefile.q.out =================================================================== --- ql/src/test/results/clientpositive/input_testsequencefile.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input_testsequencefile.q.out (working copy) @@ -16,11 +16,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -52,7 +54,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest4_sequencefile - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -87,7 +89,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest4_sequencefile + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest4_sequencefile + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest4_sequencefile SELECT src.key, src.value PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/input_testxpath.q.out =================================================================== --- ql/src/test/results/clientpositive/input_testxpath.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input_testxpath.q.out (working copy) @@ -16,11 +16,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -47,7 +49,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -82,7 +84,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM src_thrift INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint[1], src_thrift.lintstring[0].mystring, src_thrift.mstringstring['key_2'] PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/load_dyn_part1.q.out =================================================================== --- ql/src/test/results/clientpositive/load_dyn_part1.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/load_dyn_part1.q.out (working copy) @@ -41,16 +41,20 @@ STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-3 depends on stages: Stage-0 Stage-4 - Stage-10 depends on stages: Stage-2 , consists of Stage-9, Stage-8 - Stage-9 - Stage-1 depends on stages: Stage-9, Stage-8 - Stage-7 depends on stages: Stage-1 - Stage-8 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-14 depends on stages: Stage-2 , consists of Stage-11, Stage-10, Stage-12 + Stage-11 + Stage-1 depends on stages: Stage-11, Stage-10, Stage-13 + Stage-9 depends on stages: Stage-1 + Stage-10 + Stage-12 + Stage-13 depends on stages: Stage-12 STAGE PLANS: Stage: Stage-2 @@ -104,7 +108,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part2 - Stage: Stage-6 + Stage: Stage-8 Conditional Operator Stage: Stage-5 @@ -142,10 +146,29 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part1 - Stage: Stage-10 + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part1 + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-14 Conditional Operator - Stage: Stage-9 + Stage: Stage-11 Move Operator files: hdfs directory: true @@ -164,10 +187,10 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part2 - Stage: Stage-7 + Stage: Stage-9 Stats-Aggr Operator - Stage: Stage-8 + Stage: Stage-10 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### @@ -180,7 +203,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part2 + Stage: Stage-12 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part2 + Stage: Stage-13 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: from srcpart insert overwrite table nzhang_part1 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' insert overwrite table nzhang_part2 partition(ds='2008-12-31', hr) select key, value, hr where ds > '2008-04-08' Index: ql/src/test/results/clientpositive/sample2.q.out =================================================================== --- ql/src/test/results/clientpositive/sample2.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/sample2.q.out (working copy) @@ -20,11 +20,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -109,7 +111,7 @@ name: default.srcbucket name: default.srcbucket - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -208,7 +210,77 @@ name: default.dest1 name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2) s PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/input_testxpath2.q.out =================================================================== --- ql/src/test/results/clientpositive/input_testxpath2.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/input_testxpath2.q.out (working copy) @@ -16,11 +16,13 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -51,7 +53,7 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -86,7 +88,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: FROM src_thrift INSERT OVERWRITE TABLE dest1 SELECT size(src_thrift.lint), size(src_thrift.lintstring), size(src_thrift.mstringstring) where src_thrift.lint IS NOT NULL AND NOT (src_thrift.mstringstring IS NULL) PREHOOK: type: QUERY Index: ql/src/test/results/clientpositive/stats11.q.out =================================================================== --- ql/src/test/results/clientpositive/stats11.q.out (revision 1365861) +++ ql/src/test/results/clientpositive/stats11.q.out (working copy) @@ -94,16 +94,18 @@ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME srcbucket_mapjoin) a) (TOK_TABREF (TOK_TABNAME srcbucket_mapjoin_part) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME bucketmapjoin_tmp_result))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST b))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) value))) (TOK_WHERE (= (. (TOK_TABLE_OR_COL b) ds) "2008-04-08")))) STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-1 depends on stages: Stage-7 - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-9 Map Reduce Local Work Alias -> Map Local Tables: b @@ -235,7 +237,7 @@ name: default.srcbucket_mapjoin name: default.srcbucket_mapjoin - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -334,7 +336,77 @@ name: default.bucketmapjoin_tmp_result name: default.bucketmapjoin_tmp_result + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + name: default.bucketmapjoin_tmp_result + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result select /*+mapjoin(b)*/ a.key, a.value, b.value from srcbucket_mapjoin a join srcbucket_mapjoin_part b @@ -502,16 +574,18 @@ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME srcbucket_mapjoin) a) (TOK_TABREF (TOK_TABNAME srcbucket_mapjoin_part) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME bucketmapjoin_tmp_result))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST a))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) value))) (TOK_WHERE (= (. (TOK_TABLE_OR_COL b) ds) "2008-04-08")))) STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-1 depends on stages: Stage-7 - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-9 Map Reduce Local Work Alias -> Map Local Tables: a @@ -652,7 +726,7 @@ name: default.srcbucket_mapjoin_part name: default.srcbucket_mapjoin_part - Stage: Stage-5 + Stage: Stage-7 Conditional Operator Stage: Stage-4 @@ -771,7 +845,92 @@ name: default.bucketmapjoin_tmp_result name: default.bucketmapjoin_tmp_result + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numPartitions 0 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numPartitions 0 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numPartitions 0 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + name: default.bucketmapjoin_tmp_result + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result select /*+mapjoin(a)*/ a.key, a.value, b.value from srcbucket_mapjoin a join srcbucket_mapjoin_part b Index: ql/src/test/results/compiler/plan/input2.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input2.q.xml (revision 1365861) +++ ql/src/test/results/compiler/plan/input2.q.xml (working copy) @@ -6,7 +6,7 @@ - Stage-9 + Stage-11 @@ -369,6 +369,58 @@ + + + + + + + + + + + Stage-10 + + + + + + + + + + + + + + Stage-9 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -402,27 +454,16 @@ Stage-8 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + + + + @@ -464,27 +505,27 @@ - Stage-13 + Stage-17 - + - + - Stage-10 + Stage-12 - + @@ -512,19 +553,19 @@ - + - + - + - Stage-11 + Stage-13 @@ -823,6 +864,58 @@ + + + + + + + + + + + Stage-16 + + + + + + + + + + + + + + Stage-15 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -853,30 +946,19 @@ - Stage-12 + Stage-14 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + - + + + + @@ -918,27 +1000,27 @@ - Stage-17 + Stage-23 - + - + - Stage-14 + Stage-18 - + @@ -966,19 +1048,19 @@ - + - + - + - Stage-15 + Stage-19 @@ -1281,6 +1363,58 @@ + + + + + + + + + + + Stage-22 + + + + + + + + + + + + + + Stage-21 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -1320,30 +1454,19 @@ - Stage-16 + Stage-20 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + - + + + + Index: ql/src/test/results/compiler/plan/input6.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input6.q.xml (revision 1365861) +++ ql/src/test/results/compiler/plan/input6.q.xml (working copy) @@ -6,7 +6,7 @@ - Stage-7 + Stage-9 @@ -369,6 +369,58 @@ + + + + + + + + + + + Stage-8 + + + + + + + + + + + + + + Stage-7 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -402,27 +454,16 @@ Stage-6 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + + + + Index: ql/src/test/results/compiler/plan/union.q.xml =================================================================== --- ql/src/test/results/compiler/plan/union.q.xml (revision 1365861) +++ ql/src/test/results/compiler/plan/union.q.xml (working copy) @@ -6,7 +6,7 @@ - Stage-6 + Stage-8 @@ -264,6 +264,58 @@ + + + + + + + + + + + Stage-7 + + + + + + + + + + + + + + Stage-6 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -297,27 +349,16 @@ Stage-5 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + + + + Index: ql/src/test/results/compiler/plan/case_sensitivity.q.xml =================================================================== --- ql/src/test/results/compiler/plan/case_sensitivity.q.xml (revision 1365861) +++ ql/src/test/results/compiler/plan/case_sensitivity.q.xml (working copy) @@ -6,7 +6,7 @@ - Stage-7 + Stage-9 @@ -369,6 +369,58 @@ + + + + + + + + + + + Stage-8 + + + + + + + + + + + + + + Stage-7 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -402,27 +454,16 @@ Stage-6 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + + + + Index: ql/src/test/results/compiler/plan/sample3.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample3.q.xml (revision 1365861) +++ ql/src/test/results/compiler/plan/sample3.q.xml (working copy) @@ -6,7 +6,7 @@ - Stage-7 + Stage-9 @@ -369,6 +369,58 @@ + + + + + + + + + + + Stage-8 + + + + + + + + + + + + + + Stage-7 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -402,27 +454,16 @@ Stage-6 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + + + + Index: ql/src/test/results/compiler/plan/sample5.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample5.q.xml (revision 1365861) +++ ql/src/test/results/compiler/plan/sample5.q.xml (working copy) @@ -6,7 +6,7 @@ - Stage-7 + Stage-9 @@ -369,6 +369,58 @@ + + + + + + + + + + + Stage-8 + + + + + + + + + + + + + + Stage-7 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -402,27 +454,16 @@ Stage-6 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + + + + Index: ql/src/test/results/compiler/plan/sample7.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample7.q.xml (revision 1365861) +++ ql/src/test/results/compiler/plan/sample7.q.xml (working copy) @@ -6,7 +6,7 @@ - Stage-7 + Stage-9 @@ -369,6 +369,58 @@ + + + + + + + + + + + Stage-8 + + + + + + + + + + + + + + Stage-7 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -402,27 +454,16 @@ Stage-6 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + + + + Index: ql/src/test/results/compiler/plan/input1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input1.q.xml (revision 1365861) +++ ql/src/test/results/compiler/plan/input1.q.xml (working copy) @@ -6,7 +6,7 @@ - Stage-7 + Stage-9 @@ -369,6 +369,58 @@ + + + + + + + + + + + Stage-8 + + + + + + + + + + + + + + Stage-7 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -402,27 +454,16 @@ Stage-6 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + + + + Index: ql/src/test/results/compiler/plan/input3.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input3.q.xml (revision 1365861) +++ ql/src/test/results/compiler/plan/input3.q.xml (working copy) @@ -6,7 +6,7 @@ - Stage-10 + Stage-12 @@ -369,6 +369,58 @@ + + + + + + + + + + + Stage-11 + + + + + + + + + + + + + + Stage-10 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -402,27 +454,16 @@ Stage-9 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + + + + @@ -464,27 +505,27 @@ - Stage-14 + Stage-18 - + - + - Stage-11 + Stage-13 - + @@ -512,19 +553,19 @@ - + - + - + - Stage-12 + Stage-14 @@ -823,6 +864,58 @@ + + + + + + + + + + + Stage-17 + + + + + + + + + + + + + + Stage-16 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -853,30 +946,19 @@ - Stage-13 + Stage-15 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + - + + + + @@ -918,27 +1000,27 @@ - Stage-18 + Stage-24 - + - + - Stage-15 + Stage-19 - + @@ -966,19 +1048,19 @@ - + - + - + - Stage-16 + Stage-20 @@ -1281,6 +1363,58 @@ + + + + + + + + + + + Stage-23 + + + + + + + + + + + + + + Stage-22 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -1320,30 +1454,19 @@ - Stage-17 + Stage-21 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + - + + + + @@ -1385,35 +1508,35 @@ - Stage-21 + Stage-29 - + - + Stage-5 - + - + - + - Stage-19 + Stage-25 @@ -1626,6 +1749,58 @@ + + + + + + + + + + + Stage-28 + + + + + + + + + + + + + + Stage-27 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -1656,30 +1831,19 @@ - Stage-20 + Stage-26 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + - + + + + Index: ql/src/test/results/compiler/plan/input7.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input7.q.xml (revision 1365861) +++ ql/src/test/results/compiler/plan/input7.q.xml (working copy) @@ -6,7 +6,7 @@ - Stage-7 + Stage-9 @@ -369,6 +369,58 @@ + + + + + + + + + + + Stage-8 + + + + + + + + + + + + + + Stage-7 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -402,27 +454,16 @@ Stage-6 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + + + + Index: ql/src/test/results/compiler/plan/input_testsequencefile.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input_testsequencefile.q.xml (revision 1365861) +++ ql/src/test/results/compiler/plan/input_testsequencefile.q.xml (working copy) @@ -6,7 +6,7 @@ - Stage-7 + Stage-9 @@ -369,6 +369,58 @@ + + + + + + + + + + + Stage-8 + + + + + + + + + + + + + + Stage-7 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -402,27 +454,16 @@ Stage-6 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + + + + Index: ql/src/test/results/compiler/plan/input9.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input9.q.xml (revision 1365861) +++ ql/src/test/results/compiler/plan/input9.q.xml (working copy) @@ -6,7 +6,7 @@ - Stage-7 + Stage-9 @@ -369,6 +369,58 @@ + + + + + + + + + + + Stage-8 + + + + + + + + + + + + + + Stage-7 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -402,27 +454,16 @@ Stage-6 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + + + + Index: ql/src/test/results/compiler/plan/subq.q.xml =================================================================== --- ql/src/test/results/compiler/plan/subq.q.xml (revision 1365861) +++ ql/src/test/results/compiler/plan/subq.q.xml (working copy) @@ -6,7 +6,7 @@ - Stage-6 + Stage-8 @@ -264,6 +264,58 @@ + + + + + + + + + + + Stage-7 + + + + + + + + + + + + + + Stage-6 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -297,27 +349,16 @@ Stage-5 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + + + + Index: ql/src/test/results/compiler/plan/sample2.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample2.q.xml (revision 1365861) +++ ql/src/test/results/compiler/plan/sample2.q.xml (working copy) @@ -6,7 +6,7 @@ - Stage-7 + Stage-9 @@ -369,6 +369,58 @@ + + + + + + + + + + + Stage-8 + + + + + + + + + + + + + + Stage-7 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -402,27 +454,16 @@ Stage-6 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + + + + Index: ql/src/test/results/compiler/plan/sample4.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample4.q.xml (revision 1365861) +++ ql/src/test/results/compiler/plan/sample4.q.xml (working copy) @@ -6,7 +6,7 @@ - Stage-7 + Stage-9 @@ -369,6 +369,58 @@ + + + + + + + + + + + Stage-8 + + + + + + + + + + + + + + Stage-7 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -402,27 +454,16 @@ Stage-6 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + + + + Index: ql/src/test/results/compiler/plan/sample6.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample6.q.xml (revision 1365861) +++ ql/src/test/results/compiler/plan/sample6.q.xml (working copy) @@ -6,7 +6,7 @@ - Stage-7 + Stage-9 @@ -369,6 +369,58 @@ + + + + + + + + + + + Stage-8 + + + + + + + + + + + + + + Stage-7 + + + + + + + + + + + + + + true + + + #### A masked pattern was here #### + + + #### A masked pattern was here #### + + + + + + + @@ -402,27 +454,16 @@ Stage-6 - - - - - true - - - #### A masked pattern was here #### - - - #### A masked pattern was here #### - - - - + + + + Index: ql/src/test/queries/clientpositive/merge_dynamic_partition5.q =================================================================== --- ql/src/test/queries/clientpositive/merge_dynamic_partition5.q (revision 0) +++ ql/src/test/queries/clientpositive/merge_dynamic_partition5.q (working copy) @@ -0,0 +1,38 @@ +-- this is to test the case where some dynamic partitions are merged and some are moved + +create table srcpart_merge_dp like srcpart; + +create table srcpart_merge_dp_rc like srcpart; +alter table srcpart_merge_dp_rc set fileformat RCFILE; + +create table merge_dynamic_part like srcpart; +alter table merge_dynamic_part set fileformat RCFILE; + +load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11); +load data local inpath '../data/files/srcbucket21.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11); +load data local inpath '../data/files/srcbucket22.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11); +load data local inpath '../data/files/srcbucket23.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11); + +load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=12); + +insert overwrite table srcpart_merge_dp_rc partition (ds = '2008-04-08', hr) +select key, value, hr from srcpart_merge_dp where ds = '2008-04-08'; + +set hive.input.format=org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; +set hive.merge.mapfiles=true; +set hive.merge.mapredfiles=true; +set hive.merge.smallfiles.avgsize=200; +set hive.exec.compress.output=false; +set hive.exec.dynamic.partition=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +explain +insert overwrite table merge_dynamic_part partition (ds = '2008-04-08', hr) +select key, value, if(key % 100 == 0, 'a1', 'b1') as hr from srcpart_merge_dp_rc where ds = '2008-04-08'; + +insert overwrite table merge_dynamic_part partition (ds = '2008-04-08', hr) +select key, value, if(key % 100 == 0, 'a1', 'b1') as hr from srcpart_merge_dp_rc where ds = '2008-04-08'; + +show partitions merge_dynamic_part; + +select count(*) from merge_dynamic_part; Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (revision 1365861) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (working copy) @@ -265,7 +265,7 @@ ConditionalTask cndTsk = createCondTask(conf, currTask, dummyMv, cplan, fsConf.getDirName()); - LinkMoveTask(ctx, newOutput, cndTsk); + linkMoveTask(ctx, newOutput, cndTsk); } /** @@ -424,21 +424,50 @@ // // 3. add the moveTask as the children of the conditional task // - LinkMoveTask(ctx, fsOutput, cndTsk); + linkMoveTask(ctx, fsOutput, cndTsk); } - private void LinkMoveTask(GenMRProcContext ctx, FileSinkOperator newOutput, + /** + * Make the move task in the GenMRProcContext following the FileSinkOperator a dependent of all + * possible subtrees branching from the ConditionalTask. + * + * @param ctx + * @param newOutput + * @param cndTsk + */ + private void linkMoveTask(GenMRProcContext ctx, FileSinkOperator newOutput, ConditionalTask cndTsk) { List> mvTasks = ctx.getMvTask(); Task mvTask = findMoveTask(mvTasks, newOutput); for (Task tsk : cndTsk.getListTasks()) { - addDependentMoveTasks(ctx, mvTask, tsk); + linkMoveTask(ctx, mvTask, tsk); } } /** + * Follows the task tree down from task and makes all leaves parents of mvTask + * + * @param ctx + * @param mvTask + * @param task + */ + private void linkMoveTask(GenMRProcContext ctx, Task mvTask, + Task task) { + + if (task.getDependentTasks() == null || task.getDependentTasks().isEmpty()) { + // If it's a leaf, add the move task as a child + addDependentMoveTasks(ctx, mvTask, task); + } else { + // Otherwise, for each child run this method recursively + for (Task childTask : task.getDependentTasks()) { + linkMoveTask(ctx, mvTask, childTask); + } + } + } + + /** * Adds the dependencyTaskForMultiInsert in ctx as a dependent of parentTask. If mvTask is a * load table, and HIVE_MULTI_INSERT_ATOMIC_OUTPUTS is set, adds mvTask as a dependent of * dependencyTaskForMultiInsert in ctx, otherwise adds mvTask as a dependent of parentTask as @@ -551,8 +580,19 @@ Task currTask, MoveWork mvWork, MapredWork mergeWork, String inputPath) { - Task mergeTask = TaskFactory.get(mergeWork, conf); - Task moveTask = TaskFactory.get(mvWork, conf); + // There are 3 options for this ConditionalTask: + // 1) Merge the partitions + // 2) Move the partitions (i.e. don't merge the partitions) + // 3) Merge some partitions and move other partitions (i.e. merge some partitions and don't + // merge others) in this case the merge is done first followed by the move to prevent + // conflicts. + Task mergeOnlyMergeTask = TaskFactory.get(mergeWork, conf); + Task moveOnlyMoveTask = TaskFactory.get(mvWork, conf); + Task mergeAndMoveMergeTask = TaskFactory.get(mergeWork, conf); + Task mergeAndMoveMoveTask = TaskFactory.get(mvWork, conf); + + mergeAndMoveMergeTask.addDependentTask(mergeAndMoveMoveTask); + List listWorks = new ArrayList(); listWorks.add(mvWork); listWorks.add(mergeWork); @@ -560,8 +600,9 @@ ConditionalWork cndWork = new ConditionalWork(listWorks); List> listTasks = new ArrayList>(); - listTasks.add(moveTask); - listTasks.add(mergeTask); + listTasks.add(moveOnlyMoveTask); + listTasks.add(mergeOnlyMergeTask); + listTasks.add(mergeAndMoveMergeTask); ConditionalTask cndTsk = (ConditionalTask) TaskFactory.get(cndWork, conf); cndTsk.setListTasks(listTasks); Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java (revision 1365861) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java (working copy) @@ -21,13 +21,10 @@ import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -123,6 +120,7 @@ Task mvTask = ctx.getListTasks().get(0); Task mrTask = ctx.getListTasks().get(1); + Task mrAndMvTask = ctx.getListTasks().get(2); try { Path dirPath = new Path(dirName); @@ -179,11 +177,17 @@ if (doMerge) { // add the merge MR job setupMapRedWork(conf, work, trgtSize, totalSz); - resTsks.add(mrTask); // add the move task for those partitions that do not need merging - if (toMove.size() > 0) { // + if (toMove.size() > 0) { // modify the existing move task as it is already in the candidate running tasks + + // running the MoveTask and MR task in parallel may + // cause the mvTask write to /ds=1 and MR task write + // to /ds=1_1 for the same partition. + // make the MoveTask as the child of the MR Task + resTsks.add(mrAndMvTask); + MoveWork mvWork = (MoveWork) mvTask.getWork(); LoadFileDesc lfd = mvWork.getLoadFileWork(); @@ -212,21 +216,8 @@ mvWork.setLoadFileWork(null); mvWork.setLoadTableWork(null); mvWork.setMultiFilesDesc(lmfd); - - // running the MoveTask and MR task in parallel may - // cause the mvTask write to /ds=1 and MR task write - // to /ds=1_1 for the same partition. - // make the MoveTask as the child of the MR Task - List> cTasks = mrTask.getDependentTasks(); - if (cTasks != null) { - Iterator> itr = cTasks.iterator(); - while (itr.hasNext()) { - Task cld = itr.next(); - itr.remove(); - mvTask.addDependentTask(cld); - } - } - mrTask.addDependentTask(mvTask); + } else { + resTsks.add(mrTask); } } else { // add the move task resTsks.add(mvTask); @@ -246,6 +237,10 @@ } catch (IOException e) { e.printStackTrace(); } + + // Only one of the tasks should ever be added to resTsks + assert(resTsks.size() == 1); + return resTsks; }