Index: ql/src/test/results/clientpositive/notable_alias2.q.out =================================================================== --- ql/src/test/results/clientpositive/notable_alias2.q.out (revision 722634) +++ ql/src/test/results/clientpositive/notable_alias2.q.out (working copy) @@ -23,6 +23,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: rand() type: double @@ -32,8 +33,8 @@ type: int Reduce Operator Tree: Group By Operator - - expr: count(VALUE.0) + aggregations: + expr: count(VALUE.0) keys: expr: KEY.0 type: string @@ -47,11 +48,12 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/140889204/772631826.10001 + /tmp/hive-njain/202657451/1070275771.10001 Reduce Output Operator key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -61,8 +63,8 @@ type: bigint Reduce Operator Tree: Group By Operator - - expr: count(VALUE.0) + aggregations: + expr: count(VALUE.0) keys: expr: KEY.0 type: string Index: ql/src/test/results/clientpositive/join2.q.out =================================================================== --- ql/src/test/results/clientpositive/join2.q.out (revision 722634) +++ ql/src/test/results/clientpositive/join2.q.out (working copy) @@ -4,8 +4,7 @@ STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2, Stage-1 - Stage-0 depends on stages: Stage-2, Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 @@ -20,6 +19,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -36,6 +36,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -64,6 +65,7 @@ key expressions: expr: UDFToDouble(key) type: double + sort order: + Map-reduce partition columns: expr: UDFToDouble(key) type: double @@ -78,6 +80,7 @@ key expressions: expr: (UDFToDouble(0) + UDFToDouble(1)) type: double + sort order: + Map-reduce partition columns: expr: (UDFToDouble(0) + UDFToDouble(1)) type: double Index: ql/src/test/results/clientpositive/join4.q.out =================================================================== --- ql/src/test/results/clientpositive/join4.q.out (revision 722634) +++ ql/src/test/results/clientpositive/join4.q.out (working copy) @@ -24,6 +24,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -48,6 +49,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string Index: ql/src/test/results/clientpositive/input5.q.out =================================================================== --- ql/src/test/results/clientpositive/input5.q.out (revision 722634) +++ ql/src/test/results/clientpositive/input5.q.out (working copy) @@ -31,6 +31,7 @@ key expressions: expr: tkey type: string + sort order: + Map-reduce partition columns: expr: tkey type: string Index: ql/src/test/results/clientpositive/join6.q.out =================================================================== --- ql/src/test/results/clientpositive/join6.q.out (revision 722634) +++ ql/src/test/results/clientpositive/join6.q.out (working copy) @@ -24,6 +24,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -48,6 +49,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string Index: ql/src/test/results/clientpositive/join8.q.out =================================================================== --- ql/src/test/results/clientpositive/join8.q.out (revision 722634) +++ ql/src/test/results/clientpositive/join8.q.out (working copy) @@ -24,6 +24,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -48,6 +49,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string Index: ql/src/test/results/clientpositive/join10.q.out =================================================================== --- ql/src/test/results/clientpositive/join10.q.out (revision 722634) +++ ql/src/test/results/clientpositive/join10.q.out (working copy) @@ -22,6 +22,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -40,6 +41,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string Index: ql/src/test/results/clientpositive/udf3.q.out =================================================================== --- ql/src/test/results/clientpositive/udf3.q.out (revision 722634) +++ ql/src/test/results/clientpositive/udf3.q.out (working copy) @@ -13,6 +13,7 @@ src Select Operator Reduce Output Operator + sort order: Map-reduce partition columns: expr: rand() type: double @@ -22,12 +23,12 @@ type: int Reduce Operator Tree: Group By Operator - - expr: max(UDFToDouble(VALUE.0)) - expr: avg(UDFToDouble(VALUE.0)) - expr: count(VALUE.0) - expr: sum(UDFToDouble(VALUE.0)) - expr: min(UDFToDouble(VALUE.0)) + aggregations: + expr: max(UDFToDouble(VALUE.0)) + expr: avg(UDFToDouble(VALUE.0)) + expr: count(VALUE.0) + expr: sum(UDFToDouble(VALUE.0)) + expr: min(UDFToDouble(VALUE.0)) mode: partial1 File Output Operator table: @@ -38,8 +39,9 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/20731779/1245261687.10001 + /tmp/hive-njain/650928980/125435985.10001 Reduce Output Operator + sort order: tag: -1 value expressions: expr: 0 @@ -54,12 +56,12 @@ type: double Reduce Operator Tree: Group By Operator - - expr: max(VALUE.0) - expr: avg(VALUE.1) - expr: count(VALUE.2) - expr: sum(VALUE.3) - expr: min(VALUE.4) + aggregations: + expr: max(VALUE.0) + expr: avg(VALUE.1) + expr: count(VALUE.2) + expr: sum(VALUE.3) + expr: min(VALUE.4) mode: unknown Select Operator expressions: Index: ql/src/test/results/clientpositive/join12.q.out =================================================================== --- ql/src/test/results/clientpositive/join12.q.out (revision 722634) +++ ql/src/test/results/clientpositive/join12.q.out (working copy) @@ -20,6 +20,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -44,6 +45,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -68,6 +70,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string Index: ql/src/test/results/clientpositive/join14.q.out =================================================================== --- ql/src/test/results/clientpositive/join14.q.out (revision 722634) +++ ql/src/test/results/clientpositive/join14.q.out (working copy) @@ -26,6 +26,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -50,6 +51,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string Index: ql/src/test/results/clientpositive/join16.q.out =================================================================== --- ql/src/test/results/clientpositive/join16.q.out (revision 722634) +++ ql/src/test/results/clientpositive/join16.q.out (working copy) @@ -30,6 +30,7 @@ type: string expr: 1 type: string + sort order: ++ Map-reduce partition columns: expr: 0 type: string @@ -48,6 +49,7 @@ type: string expr: value type: string + sort order: ++ Map-reduce partition columns: expr: key type: string Index: ql/src/test/results/clientpositive/input17.q.out =================================================================== --- ql/src/test/results/clientpositive/input17.q.out (revision 722634) +++ ql/src/test/results/clientpositive/input17.q.out (working copy) @@ -33,6 +33,7 @@ key expressions: expr: tkey type: string + sort order: + Map-reduce partition columns: expr: tkey type: string Index: ql/src/test/results/clientpositive/groupby1.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby1.q.out (revision 722634) +++ ql/src/test/results/clientpositive/groupby1.q.out (working copy) @@ -15,6 +15,7 @@ key expressions: expr: key type: string + sort order: + Map-reduce partition columns: expr: rand() type: double @@ -24,8 +25,8 @@ type: string Reduce Operator Tree: Group By Operator - - expr: sum(UDFToDouble(VALUE.0)) + aggregations: + expr: sum(UDFToDouble(VALUE.0)) keys: expr: KEY.0 type: string @@ -39,11 +40,12 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/250776234/130709293.10001 + /tmp/hive-njain/7857726/161687564.10001 Reduce Output Operator key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -53,8 +55,8 @@ type: double Reduce Operator Tree: Group By Operator - - expr: sum(VALUE.0) + aggregations: + expr: sum(VALUE.0) keys: expr: KEY.0 type: string Index: ql/src/test/results/clientpositive/union2.q.out =================================================================== --- ql/src/test/results/clientpositive/union2.q.out (revision 0) +++ ql/src/test/results/clientpositive/union2.q.out (revision 0) @@ -0,0 +1,85 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF src s1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF s1 key) key) (TOK_SELEXPR (TOK_COLREF s1 value) value)))) (TOK_QUERY (TOK_FROM (TOK_TABREF src s2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF s2 key) key) (TOK_SELEXPR (TOK_COLREF s2 value) value))))) unionsrc)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION count 1))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + null-subquery1:unionsrc-subquery1:s1 + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + Reduce Output Operator + sort order: + Map-reduce partition columns: + expr: rand() + type: double + tag: -1 + value expressions: + expr: 1 + type: int + null-subquery2:unionsrc-subquery2:s2 + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + Reduce Output Operator + sort order: + Map-reduce partition columns: + expr: rand() + type: double + tag: -1 + value expressions: + expr: 1 + type: int + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(VALUE.0) + mode: partial1 + File Output Operator + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + /tmp/hive-njain/351706607/217456705.10002 + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: 0 + type: bigint + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(VALUE.0) + mode: unknown + Select Operator + expressions: + expr: 0 + type: bigint + File Output Operator + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +1000 Index: ql/src/test/results/clientpositive/groupby3.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby3.q.out (revision 722634) +++ ql/src/test/results/clientpositive/groupby3.q.out (working copy) @@ -19,18 +19,19 @@ key expressions: expr: substr(0, 4) type: string + sort order: + Map-reduce partition columns: expr: substr(0, 4) type: string tag: -1 Reduce Operator Tree: Group By Operator - - expr: avg(DISTINCT UDFToDouble(KEY.0)) - expr: sum(UDFToDouble(KEY.0)) - expr: avg(UDFToDouble(KEY.0)) - expr: min(UDFToDouble(KEY.0)) - expr: max(UDFToDouble(KEY.0)) + aggregations: + expr: avg(DISTINCT UDFToDouble(KEY.0)) + expr: sum(UDFToDouble(KEY.0)) + expr: avg(UDFToDouble(KEY.0)) + expr: min(UDFToDouble(KEY.0)) + expr: max(UDFToDouble(KEY.0)) mode: partial1 File Output Operator table: @@ -41,8 +42,9 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/67781830/202058716.10001 + /tmp/hive-njain/1428098224/715983841.10001 Reduce Output Operator + sort order: tag: -1 value expressions: expr: 0 @@ -57,12 +59,12 @@ type: double Reduce Operator Tree: Group By Operator - - expr: avg(VALUE.0) - expr: sum(VALUE.1) - expr: avg(VALUE.2) - expr: min(VALUE.3) - expr: max(VALUE.4) + aggregations: + expr: avg(VALUE.0) + expr: sum(VALUE.1) + expr: avg(VALUE.2) + expr: min(VALUE.3) + expr: max(VALUE.4) mode: unknown Select Operator expressions: Index: ql/src/test/results/clientpositive/groupby5.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby5.q.out (revision 722634) +++ ql/src/test/results/clientpositive/groupby5.q.out (working copy) @@ -15,6 +15,7 @@ key expressions: expr: key type: string + sort order: + Map-reduce partition columns: expr: rand() type: double @@ -24,8 +25,8 @@ type: string Reduce Operator Tree: Group By Operator - - expr: sum(UDFToDouble(VALUE.0)) + aggregations: + expr: sum(UDFToDouble(VALUE.0)) keys: expr: KEY.0 type: string @@ -39,11 +40,12 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/485297652/61546074.10001 + /tmp/hive-njain/450793281/288129670.10001 Reduce Output Operator key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -53,8 +55,8 @@ type: double Reduce Operator Tree: Group By Operator - - expr: sum(VALUE.0) + aggregations: + expr: sum(VALUE.0) keys: expr: KEY.0 type: string Index: ql/src/test/results/clientpositive/subq2.q.out =================================================================== --- ql/src/test/results/clientpositive/subq2.q.out (revision 722634) +++ ql/src/test/results/clientpositive/subq2.q.out (working copy) @@ -19,6 +19,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: rand() type: double @@ -28,8 +29,8 @@ type: int Reduce Operator Tree: Group By Operator - - expr: count(VALUE.0) + aggregations: + expr: count(VALUE.0) keys: expr: KEY.0 type: string @@ -43,11 +44,12 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/158235739/1374397779.10002 + /tmp/hive-njain/611234651/29133673.10002 Reduce Output Operator key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -57,8 +59,8 @@ type: bigint Reduce Operator Tree: Group By Operator - - expr: count(VALUE.0) + aggregations: + expr: count(VALUE.0) keys: expr: KEY.0 type: string Index: ql/src/test/results/clientpositive/input11_limit.q.out =================================================================== --- ql/src/test/results/clientpositive/input11_limit.q.out (revision 722634) +++ ql/src/test/results/clientpositive/input11_limit.q.out (working copy) @@ -22,6 +22,7 @@ type: string Limit Reduce Output Operator + sort order: tag: -1 value expressions: expr: 0 Index: ql/src/test/results/clientpositive/input14_limit.q.out =================================================================== --- ql/src/test/results/clientpositive/input14_limit.q.out (revision 722634) +++ ql/src/test/results/clientpositive/input14_limit.q.out (working copy) @@ -26,6 +26,7 @@ key expressions: expr: tkey type: string + sort order: + Map-reduce partition columns: expr: tkey type: string @@ -47,11 +48,12 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/195632265/461794176.10001 + /tmp/hive-njain/458204227/247333025.10001 Reduce Output Operator key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string Index: ql/src/test/results/clientpositive/groupby2_map.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby2_map.q.out (revision 722634) +++ ql/src/test/results/clientpositive/groupby2_map.q.out (working copy) @@ -12,9 +12,9 @@ Alias -> Map Operator Tree: src Group By Operator - - expr: count(DISTINCT substr(value, 4)) - expr: sum(UDFToDouble(substr(value, 4))) + aggregations: + expr: count(DISTINCT substr(value, 4)) + expr: sum(UDFToDouble(substr(value, 4))) keys: expr: substr(key, 0, 1) type: string @@ -27,6 +27,7 @@ type: string expr: 1 type: string + sort order: ++ Map-reduce partition columns: expr: 0 type: string @@ -40,9 +41,9 @@ type: double Reduce Operator Tree: Group By Operator - - expr: count(DISTINCT KEY.1) - expr: sum(VALUE.1) + aggregations: + expr: count(DISTINCT KEY.1) + expr: sum(VALUE.1) keys: expr: KEY.0 type: string @@ -56,11 +57,12 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/781819455/901894899.10001 + /tmp/hive-njain/951471939/220829541.10001 Reduce Output Operator key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -72,9 +74,9 @@ type: double Reduce Operator Tree: Group By Operator - - expr: count(VALUE.0) - expr: sum(VALUE.1) + aggregations: + expr: count(VALUE.0) + expr: sum(VALUE.1) keys: expr: KEY.0 type: string Index: ql/src/test/results/clientpositive/groupby4_map.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby4_map.q.out (revision 722634) +++ ql/src/test/results/clientpositive/groupby4_map.q.out (working copy) @@ -12,10 +12,11 @@ src Select Operator Group By Operator - - expr: count(1) + aggregations: + expr: count(1) mode: hash Reduce Output Operator + sort order: Map-reduce partition columns: expr: rand() type: double @@ -25,8 +26,8 @@ type: bigint Reduce Operator Tree: Group By Operator - - expr: count(VALUE.0) + aggregations: + expr: count(VALUE.0) mode: unknown Select Operator expressions: Index: ql/src/test/results/clientpositive/notable_alias1.q.out =================================================================== --- ql/src/test/results/clientpositive/notable_alias1.q.out (revision 722634) +++ ql/src/test/results/clientpositive/notable_alias1.q.out (working copy) @@ -23,6 +23,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: rand() type: double @@ -32,8 +33,8 @@ type: int Reduce Operator Tree: Group By Operator - - expr: count(VALUE.0) + aggregations: + expr: count(VALUE.0) keys: expr: KEY.0 type: string @@ -47,11 +48,12 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/252553932/208217017.10001 + /tmp/hive-njain/323467472/558965085.10001 Reduce Output Operator key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -61,8 +63,8 @@ type: bigint Reduce Operator Tree: Group By Operator - - expr: count(VALUE.0) + aggregations: + expr: count(VALUE.0) keys: expr: KEY.0 type: string Index: ql/src/test/results/clientpositive/join1.q.out =================================================================== --- ql/src/test/results/clientpositive/join1.q.out (revision 722634) +++ ql/src/test/results/clientpositive/join1.q.out (working copy) @@ -14,6 +14,7 @@ key expressions: expr: key type: string + sort order: + Map-reduce partition columns: expr: key type: string @@ -32,6 +33,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string Index: ql/src/test/results/clientpositive/join3.q.out =================================================================== --- ql/src/test/results/clientpositive/join3.q.out (revision 722634) +++ ql/src/test/results/clientpositive/join3.q.out (working copy) @@ -18,6 +18,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -30,6 +31,7 @@ key expressions: expr: key type: string + sort order: + Map-reduce partition columns: expr: key type: string @@ -48,6 +50,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string Index: ql/src/test/results/clientpositive/join5.q.out =================================================================== --- ql/src/test/results/clientpositive/join5.q.out (revision 722634) +++ ql/src/test/results/clientpositive/join5.q.out (working copy) @@ -24,6 +24,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -48,6 +49,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string Index: ql/src/test/results/clientpositive/join7.q.out =================================================================== --- ql/src/test/results/clientpositive/join7.q.out (revision 722634) +++ ql/src/test/results/clientpositive/join7.q.out (working copy) @@ -24,6 +24,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -48,6 +49,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -72,6 +74,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string Index: ql/src/test/results/clientpositive/join9.q.out =================================================================== --- ql/src/test/results/clientpositive/join9.q.out (revision 722634) +++ ql/src/test/results/clientpositive/join9.q.out (working copy) @@ -14,6 +14,7 @@ key expressions: expr: key type: string + sort order: + Map-reduce partition columns: expr: key type: string @@ -36,6 +37,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -49,10 +51,10 @@ type: string Needs Tagging: Path -> Alias: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 Path -> Partition: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Partition input format: org.apache.hadoop.mapred.TextInputFormat @@ -66,10 +68,10 @@ serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + location file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: src - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 Partition partition values: ds 2008-04-08 @@ -87,7 +89,7 @@ serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart + location file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcpart serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: srcpart Reduce Operator Tree: @@ -108,7 +110,7 @@ expr: 4 type: string File Output Operator - directory: /tmp/hive-njain/653311979.10000.insclause-0 + directory: /tmp/hive-njain/292054552.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -121,7 +123,7 @@ serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 @@ -129,7 +131,7 @@ Move Operator tables: replace: - source: /tmp/hive-njain/653311979.10000.insclause-0 + source: /tmp/hive-njain/292054552.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -142,7 +144,7 @@ serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 Index: ql/src/test/results/clientpositive/join11.q.out =================================================================== --- ql/src/test/results/clientpositive/join11.q.out (revision 722634) +++ ql/src/test/results/clientpositive/join11.q.out (working copy) @@ -20,6 +20,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -44,6 +45,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string Index: ql/src/test/results/clientpositive/join13.q.out =================================================================== --- ql/src/test/results/clientpositive/join13.q.out (revision 722634) +++ ql/src/test/results/clientpositive/join13.q.out (working copy) @@ -21,6 +21,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -45,6 +46,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -75,6 +77,7 @@ key expressions: expr: (UDFToDouble(0) + UDFToDouble(2)) type: double + sort order: + Map-reduce partition columns: expr: (UDFToDouble(0) + UDFToDouble(2)) type: double @@ -103,6 +106,7 @@ key expressions: expr: UDFToDouble(0) type: double + sort order: + Map-reduce partition columns: expr: UDFToDouble(0) type: double Index: ql/src/test/results/clientpositive/input14.q.out =================================================================== --- ql/src/test/results/clientpositive/input14.q.out (revision 722634) +++ ql/src/test/results/clientpositive/input14.q.out (working copy) @@ -25,6 +25,7 @@ key expressions: expr: tkey type: string + sort order: + Map-reduce partition columns: expr: tkey type: string Index: ql/src/test/results/clientpositive/join15.q.out =================================================================== --- ql/src/test/results/clientpositive/join15.q.out (revision 722634) +++ ql/src/test/results/clientpositive/join15.q.out (working copy) @@ -14,6 +14,7 @@ key expressions: expr: key type: string + sort order: + Map-reduce partition columns: expr: key type: string @@ -28,6 +29,7 @@ key expressions: expr: key type: string + sort order: + Map-reduce partition columns: expr: key type: string Index: ql/src/test/results/clientpositive/join17.q.out =================================================================== --- ql/src/test/results/clientpositive/join17.q.out (revision 722634) +++ ql/src/test/results/clientpositive/join17.q.out (working copy) @@ -14,6 +14,7 @@ key expressions: expr: key type: string + sort order: + Map-reduce partition columns: expr: key type: string @@ -28,6 +29,7 @@ key expressions: expr: key type: string + sort order: + Map-reduce partition columns: expr: key type: string @@ -39,9 +41,9 @@ type: string Needs Tagging: Path -> Alias: - file:/data/users/athusoo/apacheprojects/hive_local_ws1/trunk/build/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Path -> Partition: - file:/data/users/athusoo/apacheprojects/hive_local_ws1/trunk/build/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Partition input format: org.apache.hadoop.mapred.TextInputFormat @@ -55,7 +57,7 @@ serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/data/users/athusoo/apacheprojects/hive_local_ws1/trunk/build/ql/test/data/warehouse/src + location file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: src Reduce Operator Tree: @@ -76,7 +78,7 @@ expr: 3 type: string File Output Operator - directory: /tmp/hive-athusoo/156101421/211911021.10000.insclause-0 + directory: /tmp/hive-njain/562245695/1983405.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -89,7 +91,7 @@ serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/data/users/athusoo/apacheprojects/hive_local_ws1/trunk/build/ql/test/data/warehouse/dest1 + location file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 @@ -97,7 +99,7 @@ Move Operator tables: replace: - source: /tmp/hive-athusoo/156101421/211911021.10000.insclause-0 + source: /tmp/hive-njain/562245695/1983405.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -110,7 +112,7 @@ serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/data/users/athusoo/apacheprojects/hive_local_ws1/trunk/build/ql/test/data/warehouse/dest1 + location file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 Index: ql/src/test/results/clientpositive/input18.q.out =================================================================== --- ql/src/test/results/clientpositive/input18.q.out (revision 722634) +++ ql/src/test/results/clientpositive/input18.q.out (working copy) @@ -29,6 +29,7 @@ key expressions: expr: key type: string + sort order: + Map-reduce partition columns: expr: key type: string Index: ql/src/test/results/clientpositive/groupby2.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby2.q.out (revision 722634) +++ ql/src/test/results/clientpositive/groupby2.q.out (working copy) @@ -17,6 +17,7 @@ type: string expr: substr(value, 4) type: string + sort order: ++ Map-reduce partition columns: expr: substr(key, 0, 1) type: string @@ -25,9 +26,9 @@ tag: -1 Reduce Operator Tree: Group By Operator - - expr: count(DISTINCT KEY.1) - expr: sum(UDFToDouble(KEY.1)) + aggregations: + expr: count(DISTINCT KEY.1) + expr: sum(UDFToDouble(KEY.1)) keys: expr: KEY.0 type: string @@ -41,11 +42,12 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/282197599/271187968.10001 + /tmp/hive-njain/5285021/59406042.10001 Reduce Output Operator key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -57,9 +59,9 @@ type: double Reduce Operator Tree: Group By Operator - - expr: count(VALUE.0) - expr: sum(VALUE.1) + aggregations: + expr: count(VALUE.0) + expr: sum(VALUE.1) keys: expr: KEY.0 type: string Index: ql/src/test/results/clientpositive/groupby4.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby4.q.out (revision 722634) +++ ql/src/test/results/clientpositive/groupby4.q.out (working copy) @@ -19,6 +19,7 @@ key expressions: expr: substr(0, 0, 1) type: string + sort order: + Map-reduce partition columns: expr: rand() type: double @@ -38,11 +39,12 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/213545057/773995409.10001 + /tmp/hive-njain/142162733/220682119.10001 Reduce Output Operator key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string Index: ql/src/test/results/clientpositive/groupby6.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby6.q.out (revision 722634) +++ ql/src/test/results/clientpositive/groupby6.q.out (working copy) @@ -19,6 +19,7 @@ key expressions: expr: substr(0, 4, 1) type: string + sort order: + Map-reduce partition columns: expr: rand() type: double @@ -38,11 +39,12 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/911936039/66288606.10001 + /tmp/hive-njain/53979816/107916670.10001 Reduce Output Operator key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string Index: ql/src/test/results/clientpositive/input1_limit.q.out =================================================================== --- ql/src/test/results/clientpositive/input1_limit.q.out (revision 722634) +++ ql/src/test/results/clientpositive/input1_limit.q.out (working copy) @@ -24,6 +24,7 @@ type: string Limit Reduce Output Operator + sort order: tag: -1 value expressions: expr: 0 @@ -76,8 +77,9 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-athusoo/8552631/401018009.10002 + /tmp/hive-njain/842103907/482854493.10002 Reduce Output Operator + sort order: tag: -1 value expressions: expr: 0 Index: ql/src/test/results/clientpositive/input3_limit.q.out =================================================================== --- ql/src/test/results/clientpositive/input3_limit.q.out (revision 722634) +++ ql/src/test/results/clientpositive/input3_limit.q.out (working copy) @@ -18,6 +18,7 @@ type: string Limit Reduce Output Operator + sort order: tag: -1 value expressions: expr: 0 Index: ql/src/test/results/clientpositive/groupby1_map.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby1_map.q.out (revision 722634) +++ ql/src/test/results/clientpositive/groupby1_map.q.out (working copy) @@ -12,8 +12,8 @@ Alias -> Map Operator Tree: src Group By Operator - - expr: sum(UDFToDouble(substr(value, 4))) + aggregations: + expr: sum(UDFToDouble(substr(value, 4))) keys: expr: key type: string @@ -22,6 +22,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: rand() type: double @@ -31,8 +32,8 @@ type: double Reduce Operator Tree: Group By Operator - - expr: sum(VALUE.0) + aggregations: + expr: sum(VALUE.0) keys: expr: KEY.0 type: string @@ -46,11 +47,12 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/126291708/5299613.10001 + /tmp/hive-njain/871552050/153342031.10001 Reduce Output Operator key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -60,8 +62,8 @@ type: double Reduce Operator Tree: Group By Operator - - expr: sum(VALUE.0) + aggregations: + expr: sum(VALUE.0) keys: expr: KEY.0 type: string Index: ql/src/test/results/clientpositive/groupby1_limit.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby1_limit.q.out (revision 722634) +++ ql/src/test/results/clientpositive/groupby1_limit.q.out (working copy) @@ -16,6 +16,7 @@ key expressions: expr: key type: string + sort order: + Map-reduce partition columns: expr: rand() type: double @@ -25,8 +26,8 @@ type: string Reduce Operator Tree: Group By Operator - - expr: sum(UDFToDouble(VALUE.0)) + aggregations: + expr: sum(UDFToDouble(VALUE.0)) keys: expr: KEY.0 type: string @@ -40,11 +41,12 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-athusoo/28951539/40025562.10002 + /tmp/hive-njain/324923626/1284785232.10001 Reduce Output Operator key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -54,8 +56,8 @@ type: double Reduce Operator Tree: Group By Operator - - expr: sum(VALUE.0) + aggregations: + expr: sum(VALUE.0) keys: expr: KEY.0 type: string @@ -76,8 +78,9 @@ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-athusoo/28951539/40025562.10001 + /tmp/hive-njain/324923626/1284785232.10002 Reduce Output Operator + sort order: tag: -1 value expressions: expr: 0 Index: ql/src/test/results/clientpositive/groupby3_map.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby3_map.q.out (revision 722634) +++ ql/src/test/results/clientpositive/groupby3_map.q.out (working copy) @@ -16,12 +16,12 @@ expr: value type: string Group By Operator - - expr: avg(DISTINCT UDFToDouble(substr(0, 4))) - expr: sum(UDFToDouble(substr(0, 4))) - expr: avg(UDFToDouble(substr(0, 4))) - expr: min(UDFToDouble(substr(0, 4))) - expr: max(UDFToDouble(substr(0, 4))) + aggregations: + expr: avg(DISTINCT UDFToDouble(substr(0, 4))) + expr: sum(UDFToDouble(substr(0, 4))) + expr: avg(UDFToDouble(substr(0, 4))) + expr: min(UDFToDouble(substr(0, 4))) + expr: max(UDFToDouble(substr(0, 4))) keys: expr: substr(0, 4) type: string @@ -30,6 +30,7 @@ key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -47,12 +48,12 @@ type: double Reduce Operator Tree: Group By Operator - - expr: avg(DISTINCT UDFToDouble(KEY.0)) - expr: sum(VALUE.1) - expr: avg(VALUE.2) - expr: min(VALUE.3) - expr: max(VALUE.4) + aggregations: + expr: avg(DISTINCT UDFToDouble(KEY.0)) + expr: sum(VALUE.1) + expr: avg(VALUE.2) + expr: min(VALUE.3) + expr: max(VALUE.4) mode: partial2 File Output Operator table: @@ -63,8 +64,9 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/130240402/83496104.10001 + /tmp/hive-njain/82630066/2591068.10001 Reduce Output Operator + sort order: tag: -1 value expressions: expr: 0 @@ -79,12 +81,12 @@ type: double Reduce Operator Tree: Group By Operator - - expr: avg(VALUE.0) - expr: sum(VALUE.1) - expr: avg(VALUE.2) - expr: min(VALUE.3) - expr: max(VALUE.4) + aggregations: + expr: avg(VALUE.0) + expr: sum(VALUE.1) + expr: avg(VALUE.2) + expr: min(VALUE.3) + expr: max(VALUE.4) mode: unknown Select Operator expressions: Index: ql/src/test/results/clientpositive/groupby2_limit.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby2_limit.q.out (revision 722634) +++ ql/src/test/results/clientpositive/groupby2_limit.q.out (working copy) @@ -15,6 +15,7 @@ key expressions: expr: key type: string + sort order: + Map-reduce partition columns: expr: rand() type: double @@ -24,8 +25,8 @@ type: string Reduce Operator Tree: Group By Operator - - expr: sum(UDFToDouble(VALUE.0)) + aggregations: + expr: sum(UDFToDouble(VALUE.0)) keys: expr: KEY.0 type: string @@ -39,11 +40,12 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/4007501/112626006.10002 + /tmp/hive-njain/635379378/60676453.10002 Reduce Output Operator key expressions: expr: 0 type: string + sort order: + Map-reduce partition columns: expr: 0 type: string @@ -53,8 +55,8 @@ type: double Reduce Operator Tree: Group By Operator - - expr: sum(VALUE.0) + aggregations: + expr: sum(VALUE.0) keys: expr: KEY.0 type: string Index: ql/src/test/results/clientpositive/groupby5_map.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby5_map.q.out (revision 722634) +++ ql/src/test/results/clientpositive/groupby5_map.q.out (working copy) @@ -15,10 +15,11 @@ expr: key type: string Group By Operator - - expr: sum(UDFToDouble(0)) + aggregations: + expr: sum(UDFToDouble(0)) mode: hash Reduce Output Operator + sort order: Map-reduce partition columns: expr: rand() type: double @@ -28,8 +29,8 @@ type: double Reduce Operator Tree: Group By Operator - - expr: sum(VALUE.0) + aggregations: + expr: sum(VALUE.0) mode: unknown Select Operator expressions: Index: ql/src/test/results/compiler/plan/join2.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join2.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/join2.q.xml (working copy) @@ -8,7 +8,7 @@ - + Stage-1 @@ -17,9 +17,6 @@ - - - @@ -38,7 +35,7 @@ true - /tmp/hive-njain/303286777.10000.insclause-0 + /tmp/hive-njain/441145354.10000.insclause-0 @@ -87,7 +84,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -122,7 +119,7 @@ src3 - + @@ -193,6 +190,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { double joinkey0} @@ -271,7 +272,7 @@ - + @@ -459,6 +460,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { double joinkey0} @@ -542,7 +547,7 @@ - /tmp/hive-njain/547407703/58605415.10001 + /tmp/hive-njain/709768097/233295454.10001 @@ -701,6 +706,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -795,7 +804,7 @@ - + @@ -913,6 +922,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -1010,7 +1023,7 @@ - + @@ -1142,7 +1155,7 @@ - /tmp/hive-njain/547407703/58605415.10001 + /tmp/hive-njain/709768097/233295454.10001 $INTNAME @@ -1150,7 +1163,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src src3 @@ -1162,7 +1175,7 @@ - /tmp/hive-njain/547407703/58605415.10001 + /tmp/hive-njain/709768097/233295454.10001 @@ -1170,7 +1183,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -1222,7 +1235,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -1248,7 +1261,7 @@ - /tmp/hive-njain/303286777.10000.insclause-0 + /tmp/hive-njain/441145354.10000.insclause-0 @@ -1483,9 +1496,6 @@ - - - @@ -1497,11 +1507,11 @@ src2 - + src1 - + @@ -1514,7 +1524,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src src2 @@ -1529,7 +1539,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -1581,7 +1591,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/input2.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input2.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/input2.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-njain/96064841/341873759.10000.insclause-0 + /tmp/hive-njain/104100889/1367774772.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -100,7 +100,7 @@ true - /tmp/hive-njain/96064841/341873759.10001.insclause-1 + /tmp/hive-njain/104100889/1367774772.10001.insclause-1 @@ -149,7 +149,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest2 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest2 @@ -178,7 +178,7 @@ true - /tmp/hive-njain/96064841/341873759.10002.insclause-2 + /tmp/hive-njain/104100889/1367774772.10002.insclause-2 @@ -231,7 +231,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest3 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest3 @@ -259,7 +259,7 @@ src - + @@ -275,7 +275,7 @@ - /tmp/hive-njain/96064841/341873759.10000.insclause-0 + /tmp/hive-njain/104100889/1367774772.10000.insclause-0 @@ -436,7 +436,7 @@ - + @@ -483,7 +483,7 @@ - /tmp/hive-njain/96064841/341873759.10001.insclause-1 + /tmp/hive-njain/104100889/1367774772.10001.insclause-1 @@ -703,7 +703,7 @@ - + @@ -729,7 +729,7 @@ - /tmp/hive-njain/96064841/341873759.10002.insclause-2 + /tmp/hive-njain/104100889/1367774772.10002.insclause-2 @@ -875,7 +875,7 @@ - + @@ -904,7 +904,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src src @@ -916,7 +916,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -968,7 +968,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/join3.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join3.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/join3.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-njain/1136932164.10000.insclause-0 + /tmp/hive-njain/268045919.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -108,7 +108,7 @@ src2 - + @@ -155,6 +155,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -270,7 +274,7 @@ - + @@ -328,7 +332,7 @@ src3 - + @@ -367,6 +371,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -445,7 +453,7 @@ - + @@ -513,7 +521,7 @@ src1 - + @@ -556,6 +564,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -668,7 +680,7 @@ - + @@ -735,7 +747,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src src2 @@ -753,7 +765,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -805,7 +817,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -831,7 +843,7 @@ - /tmp/hive-njain/1136932164.10000.insclause-0 + /tmp/hive-njain/268045919.10000.insclause-0 Index: ql/src/test/results/compiler/plan/input3.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input3.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/input3.q.xml (working copy) @@ -28,7 +28,7 @@ true - /tmp/hive-njain/346624347/1822685820.10003.insclause-3 + /tmp/hive-njain/44210915/741228445.10003.insclause-3 ../../../../build/contrib/hive/ql/test/data/warehouse/dest4.out @@ -48,7 +48,7 @@ true - /tmp/hive-njain/346624347/1822685820.10000.insclause-0 + /tmp/hive-njain/44210915/741228445.10000.insclause-0 @@ -97,7 +97,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -117,7 +117,7 @@ true - /tmp/hive-njain/346624347/1822685820.10001.insclause-1 + /tmp/hive-njain/44210915/741228445.10001.insclause-1 @@ -166,7 +166,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest2 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest2 @@ -195,7 +195,7 @@ true - /tmp/hive-njain/346624347/1822685820.10002.insclause-2 + /tmp/hive-njain/44210915/741228445.10002.insclause-2 @@ -248,7 +248,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest3 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest3 @@ -276,7 +276,7 @@ src - + @@ -292,7 +292,7 @@ - /tmp/hive-njain/346624347/1822685820.10000.insclause-0 + /tmp/hive-njain/44210915/741228445.10000.insclause-0 @@ -453,7 +453,7 @@ - + @@ -500,7 +500,7 @@ - /tmp/hive-njain/346624347/1822685820.10001.insclause-1 + /tmp/hive-njain/44210915/741228445.10001.insclause-1 @@ -720,7 +720,7 @@ - + @@ -746,7 +746,7 @@ - /tmp/hive-njain/346624347/1822685820.10002.insclause-2 + /tmp/hive-njain/44210915/741228445.10002.insclause-2 @@ -966,7 +966,7 @@ - + @@ -992,7 +992,7 @@ - /tmp/hive-njain/346624347/1822685820.10003.insclause-3 + /tmp/hive-njain/44210915/741228445.10003.insclause-3 @@ -1140,7 +1140,7 @@ - + @@ -1169,7 +1169,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src src @@ -1181,7 +1181,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -1233,7 +1233,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/join4.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join4.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/join4.q.xml (working copy) @@ -10,7 +10,7 @@ c:b:src2 - + @@ -61,6 +61,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -378,7 +382,7 @@ - + @@ -425,7 +429,7 @@ c:a:src1 - + @@ -472,6 +476,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -778,7 +786,7 @@ - + @@ -834,7 +842,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src c:a:src1 @@ -849,7 +857,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -901,7 +909,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -931,7 +939,7 @@ - /tmp/hive-njain/170014316/124791502.10001.insclause-0 + /tmp/hive-njain/291874807/86598252.10001.insclause-0 Index: ql/src/test/results/compiler/plan/input4.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input4.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/input4.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-njain/1463239211/50419269.10000.insclause-0 + /tmp/hive-njain/89591678/324016511.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -108,7 +108,7 @@ tmap:src - + @@ -124,7 +124,7 @@ - + @@ -159,6 +159,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string reducesinkkey0} @@ -174,7 +178,18 @@ -1 - + + + + + tkey + + + + + + + -1 @@ -379,7 +394,7 @@ - + @@ -453,7 +468,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src tmap:src @@ -465,7 +480,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -517,7 +532,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -547,7 +562,7 @@ - /tmp/hive-njain/1463239211/50419269.10000.insclause-0 + /tmp/hive-njain/89591678/324016511.10000.insclause-0 Index: ql/src/test/results/compiler/plan/join5.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join5.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/join5.q.xml (working copy) @@ -10,7 +10,7 @@ c:b:src2 - + @@ -61,6 +61,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -378,7 +382,7 @@ - + @@ -425,7 +429,7 @@ c:a:src1 - + @@ -472,6 +476,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -778,7 +786,7 @@ - + @@ -834,7 +842,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src c:a:src1 @@ -849,7 +857,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -901,7 +909,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -931,7 +939,7 @@ - /tmp/hive-njain/198286603/204778341.10001.insclause-0 + /tmp/hive-njain/1109739262/425333867.10001.insclause-0 Index: ql/src/test/results/compiler/plan/input5.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input5.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/input5.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-njain/37361661.10000.insclause-0 + /tmp/hive-njain/113180245.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -108,7 +108,7 @@ tmap:src_thrift - + @@ -128,7 +128,7 @@ - + @@ -163,6 +163,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string reducesinkkey0} @@ -178,7 +182,18 @@ -1 - + + + + + tkey + + + + + + + -1 @@ -462,7 +477,7 @@ - + @@ -587,7 +602,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src_thrift + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src_thrift tmap:src_thrift @@ -599,7 +614,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src_thrift + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src_thrift @@ -655,7 +670,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src_thrift + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src_thrift @@ -681,7 +696,7 @@ - /tmp/hive-njain/37361661.10000.insclause-0 + /tmp/hive-njain/113180245.10000.insclause-0 Index: ql/src/test/results/compiler/plan/join6.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join6.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/join6.q.xml (working copy) @@ -10,7 +10,7 @@ c:b:src2 - + @@ -61,6 +61,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -378,7 +382,7 @@ - + @@ -425,7 +429,7 @@ c:a:src1 - + @@ -472,6 +476,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -778,7 +786,7 @@ - + @@ -834,7 +842,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src c:a:src1 @@ -849,7 +857,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -901,7 +909,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -931,7 +939,7 @@ - /tmp/hive-njain/113474329/1108575385.10001.insclause-0 + /tmp/hive-njain/166521027/637281771.10001.insclause-0 Index: ql/src/test/results/compiler/plan/input_testxpath2.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input_testxpath2.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/input_testxpath2.q.xml (working copy) @@ -10,7 +10,7 @@ src_thrift - + @@ -30,7 +30,7 @@ - /tmp/hive-njain/709347576.10001.insclause-0 + /tmp/hive-njain/610717789.10001.insclause-0 @@ -476,7 +476,7 @@ - + @@ -570,7 +570,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src_thrift + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src_thrift src_thrift @@ -582,7 +582,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src_thrift + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src_thrift @@ -638,7 +638,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src_thrift + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src_thrift Index: ql/src/test/results/compiler/plan/input6.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input6.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/input6.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-njain/958229575/254397981.10000.insclause-0 + /tmp/hive-njain/545048490/63016941.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -108,7 +108,7 @@ src1 - + @@ -124,7 +124,7 @@ - /tmp/hive-njain/958229575/254397981.10000.insclause-0 + /tmp/hive-njain/545048490/63016941.10000.insclause-0 @@ -265,7 +265,7 @@ - + @@ -315,7 +315,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src1 src1 @@ -327,7 +327,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src1 @@ -379,7 +379,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src1 Index: ql/src/test/results/compiler/plan/join7.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join7.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/join7.q.xml (working copy) @@ -10,7 +10,7 @@ c:b:src2 - + @@ -61,6 +61,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -378,7 +382,7 @@ - + @@ -425,7 +429,7 @@ c:a:src1 - + @@ -472,6 +476,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -778,7 +786,7 @@ - + @@ -825,7 +833,7 @@ c:c:src3 - + @@ -872,6 +880,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -1181,7 +1193,7 @@ - + @@ -1237,7 +1249,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src c:a:src1 @@ -1255,7 +1267,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -1307,7 +1319,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -1337,7 +1349,7 @@ - /tmp/hive-njain/604172747/190204720.10001.insclause-0 + /tmp/hive-njain/759592261/286929524.10001.insclause-0 Index: ql/src/test/results/compiler/plan/input7.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input7.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/input7.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-njain/505144482.10000.insclause-0 + /tmp/hive-njain/1794861784.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -108,7 +108,7 @@ src1 - + @@ -124,7 +124,7 @@ - /tmp/hive-njain/505144482.10000.insclause-0 + /tmp/hive-njain/1794861784.10000.insclause-0 @@ -242,7 +242,7 @@ - + @@ -303,7 +303,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src1 src1 @@ -315,7 +315,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src1 @@ -367,7 +367,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src1 Index: ql/src/test/results/compiler/plan/input_testsequencefile.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input_testsequencefile.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/input_testsequencefile.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-njain/317501800/36487063.10000.insclause-0 + /tmp/hive-njain/380325162/471311562.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest4_sequencefile + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest4_sequencefile @@ -108,7 +108,7 @@ src - + @@ -120,7 +120,7 @@ - /tmp/hive-njain/317501800/36487063.10000.insclause-0 + /tmp/hive-njain/380325162/471311562.10000.insclause-0 @@ -201,7 +201,7 @@ - + @@ -251,7 +251,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src src @@ -263,7 +263,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -315,7 +315,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/input8.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input8.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/input8.q.xml (working copy) @@ -10,7 +10,7 @@ src1 - + @@ -26,7 +26,7 @@ - /tmp/hive-njain/568957411.10001.insclause-0 + /tmp/hive-njain/590666906.10001.insclause-0 @@ -320,7 +320,7 @@ - + @@ -381,7 +381,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src1 src1 @@ -393,7 +393,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src1 @@ -445,7 +445,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src1 Index: ql/src/test/results/compiler/plan/join8.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join8.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/join8.q.xml (working copy) @@ -10,7 +10,7 @@ c:b:src2 - + @@ -61,6 +61,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -378,7 +382,7 @@ - + @@ -425,7 +429,7 @@ c:a:src1 - + @@ -472,6 +476,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -778,7 +786,7 @@ - + @@ -834,7 +842,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src c:a:src1 @@ -849,7 +857,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -901,7 +909,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -935,7 +943,7 @@ - /tmp/hive-njain/378159414/850135187.10001.insclause-0 + /tmp/hive-njain/19854159/32645100.10001.insclause-0 Index: ql/src/test/results/compiler/plan/union.q.xml =================================================================== --- ql/src/test/results/compiler/plan/union.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/union.q.xml (working copy) @@ -28,7 +28,7 @@ true - /tmp/hive-athusoo/37064530/676639755.10000.insclause-0 + /tmp/hive-njain/1238482884/612877942.10000.insclause-0 ../build/ql/test/data/warehouse/union.out @@ -55,7 +55,7 @@ null-subquery1:unioninput-subquery1:src - + @@ -67,7 +67,7 @@ - + @@ -79,7 +79,7 @@ - /tmp/hive-athusoo/37064530/676639755.10000.insclause-0 + /tmp/hive-njain/1238482884/612877942.10000.insclause-0 @@ -185,7 +185,7 @@ - + @@ -314,7 +314,7 @@ - + @@ -536,7 +536,7 @@ - + @@ -583,14 +583,14 @@ null-subquery2:unioninput-subquery2:src - + - file:/data/users/athusoo/apacheprojects/hive_local_ws1/trunk/build/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src null-subquery1:unioninput-subquery1:src @@ -605,7 +605,7 @@ - file:/data/users/athusoo/apacheprojects/hive_local_ws1/trunk/build/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -657,7 +657,7 @@ location - file:/data/users/athusoo/apacheprojects/hive_local_ws1/trunk/build/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/input9.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input9.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/input9.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-njain/510164179.10000.insclause-0 + /tmp/hive-njain/483179349.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -108,7 +108,7 @@ src1 - + @@ -128,7 +128,7 @@ - /tmp/hive-njain/510164179.10000.insclause-0 + /tmp/hive-njain/483179349.10000.insclause-0 @@ -324,7 +324,7 @@ - + @@ -374,7 +374,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src1 src1 @@ -386,7 +386,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src1 @@ -438,7 +438,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src1 Index: ql/src/test/results/compiler/plan/udf1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/udf1.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/udf1.q.xml (working copy) @@ -10,7 +10,7 @@ src - + @@ -30,7 +30,7 @@ - /tmp/hive-njain/530183626.10001.insclause-0 + /tmp/hive-njain/487094769.10001.insclause-0 @@ -1238,7 +1238,7 @@ - + @@ -1288,7 +1288,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src src @@ -1300,7 +1300,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -1352,7 +1352,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/udf4.q.xml =================================================================== --- ql/src/test/results/compiler/plan/udf4.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/udf4.q.xml (working copy) @@ -10,7 +10,7 @@ dest1 - + @@ -26,7 +26,7 @@ - /tmp/hive-njain/126141838.10001.insclause-0 + /tmp/hive-njain/1170725444.10001.insclause-0 @@ -948,7 +948,7 @@ - + @@ -998,7 +998,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 dest1 @@ -1010,7 +1010,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -1062,7 +1062,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 Index: ql/src/test/results/compiler/plan/input_testxpath.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input_testxpath.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/input_testxpath.q.xml (working copy) @@ -10,7 +10,7 @@ src_thrift - + @@ -26,7 +26,7 @@ - /tmp/hive-njain/493664790.10001.insclause-0 + /tmp/hive-njain/1163598770.10001.insclause-0 @@ -288,7 +288,7 @@ - + @@ -413,7 +413,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src_thrift + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src_thrift src_thrift @@ -425,7 +425,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src_thrift + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src_thrift @@ -481,7 +481,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src_thrift + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src_thrift Index: ql/src/test/results/compiler/plan/input_part1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input_part1.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/input_part1.q.xml (working copy) @@ -10,7 +10,7 @@ srcpart - + @@ -26,7 +26,7 @@ - /tmp/hive-njain/133993466/535789358.10001.insclause-0 + /tmp/hive-njain/381790525/715994016.10001.insclause-0 @@ -394,7 +394,7 @@ - + @@ -464,7 +464,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 srcpart @@ -476,7 +476,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 @@ -541,7 +541,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcpart Index: ql/src/test/results/compiler/plan/groupby1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby1.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/groupby1.q.xml (working copy) @@ -35,7 +35,7 @@ true - /tmp/hive-njain/255951081/1610738340.10000.insclause-0 + /tmp/hive-njain/1734876980/99648103.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -118,7 +118,7 @@ - /tmp/hive-njain/255951081/1610738340.10001 + /tmp/hive-njain/1734876980/99648103.10001 @@ -161,6 +161,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string reducesinkkey0} @@ -241,7 +245,7 @@ - /tmp/hive-njain/255951081/1610738340.10001 + /tmp/hive-njain/1734876980/99648103.10001 @@ -423,6 +427,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string reducesinkkey0} @@ -552,7 +560,7 @@ - + @@ -672,10 +680,10 @@ - /tmp/hive-njain/255951081/1610738340.10001 + /tmp/hive-njain/1734876980/99648103.10001 - /tmp/hive-njain/255951081/1610738340.10001 + /tmp/hive-njain/1734876980/99648103.10001 @@ -684,7 +692,7 @@ - /tmp/hive-njain/255951081/1610738340.10001 + /tmp/hive-njain/1734876980/99648103.10001 @@ -706,7 +714,7 @@ - /tmp/hive-njain/255951081/1610738340.10000.insclause-0 + /tmp/hive-njain/1734876980/99648103.10000.insclause-0 @@ -905,7 +913,7 @@ src - + @@ -915,7 +923,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src src @@ -927,7 +935,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -979,7 +987,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/groupby2.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby2.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/groupby2.q.xml (working copy) @@ -20,7 +20,7 @@ - /tmp/hive-njain/16836316/818176241.10002 + /tmp/hive-njain/568944209/29322851.10002 @@ -63,6 +63,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string reducesinkkey0} @@ -157,7 +161,7 @@ - /tmp/hive-njain/16836316/818176241.10002 + /tmp/hive-njain/568944209/29322851.10002 @@ -474,6 +478,10 @@ binary_sortable_table + serialization.sort.order + ++ + + serialization.ddl struct binary_sortable_table { string reducesinkkey0, string reducesinkkey1} @@ -531,7 +539,7 @@ - + @@ -661,10 +669,10 @@ - /tmp/hive-njain/16836316/818176241.10002 + /tmp/hive-njain/568944209/29322851.10002 - /tmp/hive-njain/16836316/818176241.10002 + /tmp/hive-njain/568944209/29322851.10002 @@ -673,7 +681,7 @@ - /tmp/hive-njain/16836316/818176241.10002 + /tmp/hive-njain/568944209/29322851.10002 @@ -695,7 +703,7 @@ - /tmp/hive-njain/16836316/818176241.10001.insclause-0 + /tmp/hive-njain/568944209/29322851.10001.insclause-0 @@ -1028,7 +1036,7 @@ src - + @@ -1038,7 +1046,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src src @@ -1050,7 +1058,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -1102,7 +1110,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/subq.q.xml =================================================================== --- ql/src/test/results/compiler/plan/subq.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/subq.q.xml (working copy) @@ -28,7 +28,7 @@ true - /tmp/hive-athusoo/853032200/122389140.10000.insclause-0 + /tmp/hive-njain/276583968/36258042.10000.insclause-0 ../build/ql/test/data/warehouse/union.out @@ -55,7 +55,7 @@ unioninput:src - + @@ -75,7 +75,7 @@ - /tmp/hive-athusoo/853032200/122389140.10000.insclause-0 + /tmp/hive-njain/276583968/36258042.10000.insclause-0 @@ -328,7 +328,7 @@ - + @@ -378,7 +378,7 @@ - file:/data/users/athusoo/apacheprojects/hive_local_ws1/trunk/build/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src unioninput:src @@ -390,7 +390,7 @@ - file:/data/users/athusoo/apacheprojects/hive_local_ws1/trunk/build/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -442,7 +442,7 @@ location - file:/data/users/athusoo/apacheprojects/hive_local_ws1/trunk/build/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/groupby3.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby3.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/groupby3.q.xml (working copy) @@ -20,7 +20,7 @@ - /tmp/hive-njain/6206243/1572612461.10002 + /tmp/hive-njain/332856862/73535516.10002 @@ -48,6 +48,10 @@ binary_sortable_table + serialization.sort.order + + + serialization.ddl struct binary_sortable_table { } @@ -172,7 +176,7 @@ - /tmp/hive-njain/6206243/1572612461.10002 + /tmp/hive-njain/332856862/73535516.10002 @@ -597,6 +601,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string reducesinkkey0} @@ -683,7 +691,7 @@ - + @@ -845,10 +853,10 @@ - /tmp/hive-njain/6206243/1572612461.10002 + /tmp/hive-njain/332856862/73535516.10002 - /tmp/hive-njain/6206243/1572612461.10002 + /tmp/hive-njain/332856862/73535516.10002 @@ -857,7 +865,7 @@ - /tmp/hive-njain/6206243/1572612461.10002 + /tmp/hive-njain/332856862/73535516.10002 @@ -879,7 +887,7 @@ - /tmp/hive-njain/453676475.10001.insclause-0 + /tmp/hive-njain/229394155.10001.insclause-0 @@ -1263,7 +1271,7 @@ src - + @@ -1273,7 +1281,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src src @@ -1285,7 +1293,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -1337,7 +1345,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/groupby4.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby4.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/groupby4.q.xml (working copy) @@ -20,7 +20,7 @@ - /tmp/hive-njain/887282678/418909568.10002 + /tmp/hive-njain/270959803/34104799.10002 @@ -63,6 +63,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string reducesinkkey0} @@ -128,7 +132,7 @@ - /tmp/hive-njain/887282678/418909568.10002 + /tmp/hive-njain/270959803/34104799.10002 @@ -308,6 +312,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string reducesinkkey0} @@ -418,7 +426,7 @@ - + @@ -540,10 +548,10 @@ - /tmp/hive-njain/887282678/418909568.10002 + /tmp/hive-njain/270959803/34104799.10002 - /tmp/hive-njain/887282678/418909568.10002 + /tmp/hive-njain/270959803/34104799.10002 @@ -552,7 +560,7 @@ - /tmp/hive-njain/887282678/418909568.10002 + /tmp/hive-njain/270959803/34104799.10002 @@ -574,7 +582,7 @@ - /tmp/hive-njain/261034330.10001.insclause-0 + /tmp/hive-njain/648245175.10001.insclause-0 @@ -743,7 +751,7 @@ src - + @@ -753,7 +761,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src src @@ -765,7 +773,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -817,7 +825,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/groupby5.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby5.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/groupby5.q.xml (working copy) @@ -20,7 +20,7 @@ - /tmp/hive-njain/261145182/229076034.10002 + /tmp/hive-njain/25539710/241360638.10002 @@ -63,6 +63,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string reducesinkkey0} @@ -143,7 +147,7 @@ - /tmp/hive-njain/261145182/229076034.10002 + /tmp/hive-njain/25539710/241360638.10002 @@ -325,6 +329,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string reducesinkkey0} @@ -454,7 +462,7 @@ - + @@ -574,10 +582,10 @@ - /tmp/hive-njain/261145182/229076034.10002 + /tmp/hive-njain/25539710/241360638.10002 - /tmp/hive-njain/261145182/229076034.10002 + /tmp/hive-njain/25539710/241360638.10002 @@ -586,7 +594,7 @@ - /tmp/hive-njain/261145182/229076034.10002 + /tmp/hive-njain/25539710/241360638.10002 @@ -608,7 +616,7 @@ - /tmp/hive-njain/261145182/229076034.10001.insclause-0 + /tmp/hive-njain/25539710/241360638.10001.insclause-0 @@ -829,7 +837,7 @@ src - + @@ -839,7 +847,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src src @@ -851,7 +859,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -903,7 +911,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/groupby6.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby6.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/groupby6.q.xml (working copy) @@ -20,7 +20,7 @@ - /tmp/hive-njain/14395907/27060646.10002 + /tmp/hive-njain/1062373643/636855904.10002 @@ -63,6 +63,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string reducesinkkey0} @@ -128,7 +132,7 @@ - /tmp/hive-njain/14395907/27060646.10002 + /tmp/hive-njain/1062373643/636855904.10002 @@ -308,6 +312,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string reducesinkkey0} @@ -418,7 +426,7 @@ - + @@ -540,10 +548,10 @@ - /tmp/hive-njain/14395907/27060646.10002 + /tmp/hive-njain/1062373643/636855904.10002 - /tmp/hive-njain/14395907/27060646.10002 + /tmp/hive-njain/1062373643/636855904.10002 @@ -552,7 +560,7 @@ - /tmp/hive-njain/14395907/27060646.10002 + /tmp/hive-njain/1062373643/636855904.10002 @@ -574,7 +582,7 @@ - /tmp/hive-njain/786616721.10001.insclause-0 + /tmp/hive-njain/786081691.10001.insclause-0 @@ -743,7 +751,7 @@ src - + @@ -753,7 +761,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src src @@ -765,7 +773,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -817,7 +825,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/case_sensitivity.q.xml =================================================================== --- ql/src/test/results/compiler/plan/case_sensitivity.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/case_sensitivity.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-njain/14809281.10000.insclause-0 + /tmp/hive-njain/1375367896.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -108,7 +108,7 @@ src_thrift - + @@ -128,7 +128,7 @@ - /tmp/hive-njain/14809281.10000.insclause-0 + /tmp/hive-njain/1375367896.10000.insclause-0 @@ -425,7 +425,7 @@ - + @@ -526,7 +526,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src_thrift + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src_thrift src_thrift @@ -538,7 +538,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src_thrift + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src_thrift @@ -594,7 +594,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src_thrift + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src_thrift Index: ql/src/test/results/compiler/plan/sample1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample1.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/sample1.q.xml (working copy) @@ -10,7 +10,7 @@ s - + @@ -30,7 +30,7 @@ - /tmp/hive-njain/57674480/1824861482.10001.insclause-0 + /tmp/hive-njain/425404034/89912765.10001.insclause-0 @@ -548,7 +548,7 @@ - + @@ -570,7 +570,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 s @@ -582,7 +582,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 @@ -647,7 +647,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcpart Index: ql/src/test/results/compiler/plan/sample2.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample2.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/sample2.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-njain/186273508/11889374.10000.insclause-0 + /tmp/hive-njain/20107464/1379933163.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -108,7 +108,7 @@ s - + @@ -120,7 +120,7 @@ - /tmp/hive-njain/186273508/11889374.10000.insclause-0 + /tmp/hive-njain/20107464/1379933163.10000.insclause-0 @@ -204,7 +204,7 @@ - + @@ -254,7 +254,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket/kv1.txt s @@ -266,7 +266,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket/kv1.txt @@ -322,7 +322,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket Index: ql/src/test/results/compiler/plan/sample3.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample3.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/sample3.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-njain/164819642/256272558.10000.insclause-0 + /tmp/hive-njain/1789469168/381631340.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -108,7 +108,7 @@ s - + @@ -124,7 +124,7 @@ - /tmp/hive-njain/164819642/256272558.10000.insclause-0 + /tmp/hive-njain/1789469168/381631340.10000.insclause-0 @@ -396,7 +396,7 @@ - + @@ -439,7 +439,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket s @@ -451,7 +451,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket @@ -507,7 +507,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket Index: ql/src/test/results/compiler/plan/sample4.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample4.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/sample4.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-njain/85259061/94539507.10000.insclause-0 + /tmp/hive-njain/1853542808/45744403.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -108,7 +108,7 @@ s - + @@ -120,7 +120,7 @@ - /tmp/hive-njain/85259061/94539507.10000.insclause-0 + /tmp/hive-njain/1853542808/45744403.10000.insclause-0 @@ -204,7 +204,7 @@ - + @@ -254,7 +254,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket/kv1.txt s @@ -266,7 +266,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket/kv1.txt @@ -322,7 +322,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket Index: ql/src/test/results/compiler/plan/sample5.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample5.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/sample5.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-njain/423510659/355253855.10000.insclause-0 + /tmp/hive-njain/824806517/591440948.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -108,7 +108,7 @@ s - + @@ -124,7 +124,7 @@ - /tmp/hive-njain/423510659/355253855.10000.insclause-0 + /tmp/hive-njain/824806517/591440948.10000.insclause-0 @@ -383,7 +383,7 @@ - + @@ -426,7 +426,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket s @@ -438,7 +438,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket @@ -494,7 +494,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket Index: ql/src/test/results/compiler/plan/sample6.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample6.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/sample6.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-njain/404713103/151871838.10000.insclause-0 + /tmp/hive-njain/814020855/624762724.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -108,7 +108,7 @@ s - + @@ -124,7 +124,7 @@ - /tmp/hive-njain/404713103/151871838.10000.insclause-0 + /tmp/hive-njain/814020855/624762724.10000.insclause-0 @@ -383,7 +383,7 @@ - + @@ -426,7 +426,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket/kv1.txt s @@ -438,7 +438,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket/kv1.txt @@ -494,7 +494,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket Index: ql/src/test/results/compiler/plan/sample7.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample7.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/sample7.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-njain/1242963668/223671734.10000.insclause-0 + /tmp/hive-njain/475551879/391349136.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -108,7 +108,7 @@ s - + @@ -128,7 +128,7 @@ - /tmp/hive-njain/1242963668/223671734.10000.insclause-0 + /tmp/hive-njain/475551879/391349136.10000.insclause-0 @@ -477,7 +477,7 @@ - + @@ -499,7 +499,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket/kv1.txt s @@ -511,7 +511,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket/kv1.txt @@ -567,7 +567,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket Index: ql/src/test/results/compiler/plan/cast1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/cast1.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/cast1.q.xml (working copy) @@ -10,7 +10,7 @@ src - + @@ -30,7 +30,7 @@ - /tmp/hive-njain/168127484.10001.insclause-0 + /tmp/hive-njain/802827123.10001.insclause-0 @@ -666,7 +666,7 @@ - + @@ -716,7 +716,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src src @@ -728,7 +728,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -780,7 +780,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/join1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join1.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/join1.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-njain/810786628.10000.insclause-0 + /tmp/hive-njain/605771515.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -108,7 +108,7 @@ src2 - + @@ -151,6 +151,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -229,7 +233,7 @@ - + @@ -297,7 +301,7 @@ src1 - + @@ -340,6 +344,10 @@ binary_sortable_table + serialization.sort.order + + + + serialization.ddl struct binary_sortable_table { string joinkey0} @@ -452,7 +460,7 @@ - + @@ -519,7 +527,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src src2 @@ -534,7 +542,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -586,7 +594,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -612,7 +620,7 @@ - /tmp/hive-njain/810786628.10000.insclause-0 + /tmp/hive-njain/605771515.10000.insclause-0 Index: ql/src/test/results/compiler/plan/input1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input1.q.xml (revision 722634) +++ ql/src/test/results/compiler/plan/input1.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-njain/1266815017/75925705.10000.insclause-0 + /tmp/hive-njain/202548334/480112542.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1 @@ -108,7 +108,7 @@ src - + @@ -124,7 +124,7 @@ - /tmp/hive-njain/1266815017/75925705.10000.insclause-0 + /tmp/hive-njain/202548334/480112542.10000.insclause-0 @@ -282,7 +282,7 @@ - + @@ -332,7 +332,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src src @@ -344,7 +344,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src @@ -396,7 +396,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src Index: ql/src/test/queries/clientpositive/union2.q =================================================================== --- ql/src/test/queries/clientpositive/union2.q (revision 0) +++ ql/src/test/queries/clientpositive/union2.q (revision 0) @@ -0,0 +1,6 @@ +explain + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION ALL + select s2.key as key, s2.value as value from src s2) unionsrc; + +select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION ALL + select s2.key as key, s2.value as value from src s2) unionsrc; Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorProcessorContext.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorProcessorContext.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorProcessorContext.java (revision 0) @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer; + +/** + * Operator Processor Context + */ +public abstract class OperatorProcessorContext { +} Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java (revision 0) @@ -0,0 +1,336 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer; + +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Set; +import java.util.Stack; +import java.io.Serializable; +import java.io.File; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import org.apache.hadoop.hive.ql.exec.FileSinkOperator; +import org.apache.hadoop.hive.ql.exec.TableScanOperator; +import org.apache.hadoop.hive.ql.exec.JoinOperator; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.exec.OperatorFactory; +import org.apache.hadoop.hive.ql.plan.mapredWork; +import org.apache.hadoop.hive.ql.plan.reduceSinkDesc; +import org.apache.hadoop.hive.ql.plan.tableDesc; +import org.apache.hadoop.hive.ql.plan.partitionDesc; +import org.apache.hadoop.hive.ql.plan.fileSinkDesc; +import org.apache.hadoop.hive.ql.plan.PlanUtils; +import org.apache.hadoop.hive.ql.metadata.*; +import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.fs.Path; + +/** + * Processor Context for creating map reduce task. Walk the tree in a DFS manner and process the nodes. Some state is + * maintained about the current nodes visited so far. + */ +public class GenMRProcContext extends OperatorProcessorContext { + + /** + * GenMapRedCtx is used to keep track of the current state. + */ + public static class GenMapRedCtx { + Task currTask; + Operator currTopOp; + String currAliasId; + + /** + * @param currTask the current task + * @param currTopOp the current top operator being traversed + * @param currAliasId the current alias for the to operator + */ + public GenMapRedCtx (Task currTask, + Operator currTopOp, + String currAliasId) { + this.currTask = currTask; + this.currTopOp = currTopOp; + this.currAliasId = currAliasId; + } + + /** + * @return current task + */ + public Task getCurrTask() { + return currTask; + } + + /** + * @return current top operator + */ + public Operator getCurrTopOp() { + return currTopOp; + } + + /** + * @return current alias + */ + public String getCurrAliasId() { + return currAliasId; + } + } + + private HashMap, Task> opTaskMap; + private List> seenOps; + + private ParseContext parseCtx; + private Task mvTask; + private List> rootTasks; + private String scratchDir; + private int randomid; + private int pathid; + + private Map, GenMapRedCtx> mapCurrCtx; + private Task currTask; + private Operator currTopOp; + private String currAliasId; + private List> rootOps; + + /** + * @param opTaskMap reducer to task mapping + * @param seenOps operator already visited + * @param parseCtx current parse context + * @param rootTasks root tasks for the plan + * @param mvTask the final move task + * @param scratchDir directory for temp destinations + * @param randomId identifier used for temp destinations + * @param pathId identifier used for temp destinations + * @param mapCurrCtx operator to task mappings + */ + public GenMRProcContext ( + HashMap, Task> opTaskMap, + List> seenOps, + ParseContext parseCtx, + Task mvTask, + List> rootTasks, + String scratchDir, int randomid, int pathid, + Map, GenMapRedCtx> mapCurrCtx) + { + + this.opTaskMap = opTaskMap; + this.seenOps = seenOps; + this.mvTask = mvTask; + this.parseCtx = parseCtx; + this.rootTasks = rootTasks; + this.scratchDir = scratchDir; + this.randomid = randomid; + this.pathid = pathid; + this.mapCurrCtx = mapCurrCtx; + currTask = null; + currTopOp = null; + currAliasId = null; + rootOps = new ArrayList>(); + rootOps.addAll(parseCtx.getTopOps().values()); + } + + /** + * @return reducer to task mapping + */ + public HashMap, Task> getOpTaskMap() { + return opTaskMap; + } + + /** + * @param opTaskMap reducer to task mapping + */ + public void setOpTaskMap(HashMap, Task> opTaskMap) { + this.opTaskMap = opTaskMap; + } + + /** + * @return operators already visited + */ + public List> getSeenOps() { + return seenOps; + } + + /** + * @param seenOps operators already visited + */ + public void setSeenOps(List> seenOps) { + this.seenOps = seenOps; + } + + /** + * @return top operators for tasks + */ + public List> getRootOps() { + return rootOps; + } + + /** + * @param rootOps top operators for tasks + */ + public void setRootOps(List> rootOps) { + this.rootOps = rootOps; + } + + /** + * @return current parse context + */ + public ParseContext getParseCtx() { + return parseCtx; + } + + /** + * @param parseCtx current parse context + */ + public void setParseCtx(ParseContext parseCtx) { + this.parseCtx = parseCtx; + } + + /** + * @return the final move task + */ + public Task getMvTask() { + return mvTask; + } + + /** + * @param mvTask the final move task + */ + public void setMvTask(Task mvTask) { + this.mvTask = mvTask; + } + + /** + * @return root tasks for the plan + */ + public List> getRootTasks() { + return rootTasks; + } + + /** + * @param rootTasks root tasks for the plan + */ + public void setRootTasks(List> rootTasks) { + this.rootTasks = rootTasks; + } + + /** + * @return directory for temp destinations + */ + public String getScratchDir() { + return scratchDir; + } + + /** + * @param scratchDir directory for temp destinations + */ + public void setScratchDir(String scratchDir) { + this.scratchDir = scratchDir; + } + + /** + * @return identifier used for temp destinations + */ + public int getRandomId() { + return randomid; + } + + /** + * @param randomId identifier used for temp destinations + */ + public void setRandomId(int randomid) { + this.randomid = randomid; + } + + /** + * @return identifier used for temp destinations + */ + public int getPathId() { + return pathid; + } + + /** + * @param pathid identifier used for temp destinations + */ + public void setPathId(int pathid) { + this.pathid = pathid; + } + + /** + * @return operator to task mappings + */ + public Map, GenMapRedCtx> getMapCurrCtx() { + return mapCurrCtx; + } + + /** + * @param mapCurrCtx operator to task mappings + */ + public void setMapCurrCtx(Map, GenMapRedCtx> mapCurrCtx) { + this.mapCurrCtx = mapCurrCtx; + } + + /** + * @return current task + */ + public Task getCurrTask() { + return currTask; + } + + /** + * @param currTask current task + */ + public void setCurrTask(Task currTask) { + this.currTask = currTask; + } + + /** + * @return current top operator + */ + public Operator getCurrTopOp() { + return currTopOp; + } + + /** + * @param currTopOp current top operator + */ + public void setCurrTopOp(Operator currTopOp) { + this.currTopOp = currTopOp; + } + + /** + * @return current top alias + */ + public String getCurrAliasId() { + return currAliasId; + } + + /** + * @param currAliasId current top alias + */ + public void setCurrAliasId(String currAliasId) { + this.currAliasId = currAliasId; + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (revision 0) @@ -0,0 +1,343 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer; + +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Set; +import java.util.Stack; +import java.io.Serializable; +import java.io.File; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import org.apache.hadoop.hive.ql.exec.FileSinkOperator; +import org.apache.hadoop.hive.ql.exec.TableScanOperator; +import org.apache.hadoop.hive.ql.exec.JoinOperator; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.exec.OperatorFactory; +import org.apache.hadoop.hive.ql.plan.mapredWork; +import org.apache.hadoop.hive.ql.plan.reduceSinkDesc; +import org.apache.hadoop.hive.ql.plan.tableDesc; +import org.apache.hadoop.hive.ql.plan.partitionDesc; +import org.apache.hadoop.hive.ql.plan.fileSinkDesc; +import org.apache.hadoop.hive.ql.plan.PlanUtils; +import org.apache.hadoop.hive.ql.metadata.*; +import org.apache.hadoop.hive.ql.parse.*; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * General utility common functions for the Processor to convert operator into map-reduce tasks + */ +public class GenMapRedUtils { + private static Log LOG; + + static { + LOG = LogFactory.getLog("org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils"); + } + + /** + * Initialize the current plan by adding it to root tasks + * @param op the reduce sink operator encountered + * @param opProcCtx processing context + */ + public static void initPlan(ReduceSinkOperator op, GenMRProcContext opProcCtx) throws SemanticException { + Operator reducer = op.getChildOperators().get(0); + Map, GenMapRedCtx> mapCurrCtx = opProcCtx.getMapCurrCtx(); + GenMapRedCtx mapredCtx = mapCurrCtx.get(op.getParentOperators().get(0)); + Task currTask = mapredCtx.getCurrTask(); + mapredWork plan = (mapredWork) currTask.getWork(); + HashMap, Task> opTaskMap = opProcCtx.getOpTaskMap(); + Operator currTopOp = opProcCtx.getCurrTopOp(); + + opTaskMap.put(reducer, currTask); + plan.setReducer(reducer); + reduceSinkDesc desc = (reduceSinkDesc)op.getConf(); + + // The number of reducers may be specified in the plan in some cases, or may need to be inferred + if (desc != null) { + if (desc.getNumReducers() != -1) + plan.setNumReduceTasks(new Integer(desc.getNumReducers())); + else if (desc.getInferNumReducers() == true) + plan.setInferNumReducers(true); + } + + List> rootTasks = opProcCtx.getRootTasks(); + + rootTasks.add(currTask); + if (reducer.getClass() == JoinOperator.class) + plan.setNeedsTagging(true); + + assert currTopOp != null; + List> seenOps = opProcCtx.getSeenOps(); + String currAliasId = opProcCtx.getCurrAliasId(); + + seenOps.add(currTopOp); + setTaskPlan(currAliasId, currTopOp, plan, false, opProcCtx); + + currTopOp = null; + currAliasId = null; + + opProcCtx.setCurrTask(currTask); + opProcCtx.setCurrTopOp(currTopOp); + opProcCtx.setCurrAliasId(currAliasId); + } + + /** + * Merge the current task with the task for the current reducer + * @param task for the old task for the current reducer + * @param opProcCtx processing context + */ + public static void joinPlan(Task task, GenMRProcContext opProcCtx) throws SemanticException { + Task currTask = task; + mapredWork plan = (mapredWork) currTask.getWork(); + Operator currTopOp = opProcCtx.getCurrTopOp(); + + if (currTopOp != null) { + List> seenOps = opProcCtx.getSeenOps(); + String currAliasId = opProcCtx.getCurrAliasId(); + + if (!seenOps.contains(currTopOp)) { + seenOps.add(currTopOp); + setTaskPlan(currAliasId, currTopOp, plan, false, opProcCtx); + } + currTopOp = null; + opProcCtx.setCurrTopOp(currTopOp); + } + } + + /** + * Split the current plan by creating a temporary destination + * @param op the reduce sink operator encountered + * @param opProcCtx processing context + */ + public static void splitPlan(ReduceSinkOperator op, GenMRProcContext opProcCtx) { + // Generate a new task + mapredWork cplan = getMapRedWork(); + ParseContext parseCtx = opProcCtx.getParseCtx(); + Task redTask = TaskFactory.get(cplan, parseCtx.getConf()); + Operator reducer = op.getChildOperators().get(0); + + // Add the reducer + cplan.setReducer(reducer); + reduceSinkDesc desc = (reduceSinkDesc)op.getConf(); + + if (desc.getNumReducers() != -1) + cplan.setNumReduceTasks(new Integer(desc.getNumReducers())); + else + cplan.setInferNumReducers(desc.getInferNumReducers()); + + HashMap, Task> opTaskMap = opProcCtx.getOpTaskMap(); + opTaskMap.put(reducer, redTask); + + // generate the temporary file + String scratchDir = opProcCtx.getScratchDir(); + int randomid = opProcCtx.getRandomId(); + int pathid = opProcCtx.getPathId(); + + String taskTmpDir = scratchDir + File.separator + randomid + '.' + pathid ; + pathid++; + opProcCtx.setPathId(pathid); + + Operator parent = op.getParentOperators().get(0); + tableDesc tt_desc = + PlanUtils.getBinaryTableDesc(PlanUtils.getFieldSchemasFromRowSchema(parent.getSchema(), "temporarycol")); + + // Create a file sink operator for this file name + Operator fs_op = + putOpInsertMap(OperatorFactory.get(new fileSinkDesc(taskTmpDir, tt_desc),parent.getSchema()), null, parseCtx); + + // replace the reduce child with this operator + List> childOpList = parent.getChildOperators(); + for (int pos = 0; pos < childOpList.size(); pos++) { + if (childOpList.get(pos) == op) { + childOpList.set(pos, fs_op); + break; + } + } + + List> parentOpList = new ArrayList>(); + parentOpList.add(parent); + fs_op.setParentOperators(parentOpList); + + // Add the path to alias mapping + if (cplan.getPathToAliases().get(taskTmpDir) == null) { + cplan.getPathToAliases().put(taskTmpDir, new ArrayList()); + } + + String streamDesc; + if (reducer.getClass() == JoinOperator.class) + streamDesc = "$INTNAME"; + else + streamDesc = taskTmpDir; + + cplan.getPathToAliases().get(taskTmpDir).add(streamDesc); + cplan.getPathToPartitionInfo().put(taskTmpDir, new partitionDesc(tt_desc, null)); + cplan.getAliasToWork().put(streamDesc, op); + setKeyAndValueDesc(cplan, op); + + // Make this task dependent on the current task + Task currTask = opProcCtx.getCurrTask(); + currTask.addDependentTask(redTask); + + // TODO: Allocate work to remove the temporary files and make that + // dependent on the redTask + if (reducer.getClass() == JoinOperator.class) + cplan.setNeedsTagging(true); + + Operator currTopOp = opProcCtx.getCurrTopOp(); + String currAliasId = opProcCtx.getCurrAliasId(); + + currTopOp = null; + currAliasId = null; + currTask = redTask; + + opProcCtx.setCurrTask(currTask); + opProcCtx.setCurrTopOp(currTopOp); + opProcCtx.setCurrAliasId(currAliasId); + opProcCtx.getRootOps().add(op); + } + + /** + * set the current task in the mapredWork + * @param alias_id current alias + * @param topOp the top operator of the stack + * @param plan current plan + * @param local whether you need to add to map-reduce or local work + * @param opProcCtx processing context + */ + public static void setTaskPlan(String alias_id, Operator topOp, mapredWork plan, boolean local, GenMRProcContext opProcCtx) + throws SemanticException { + ParseContext parseCtx = opProcCtx.getParseCtx(); + + if (!local) { + // Generate the map work for this alias_id + PartitionPruner pruner = parseCtx.getAliasToPruner().get(alias_id); + Set parts = null; + try { + // pass both confirmed and unknown partitions through the map-reduce framework + PartitionPruner.PrunedPartitionList partsList = pruner.prune(); + parts = partsList.getConfirmedPartns(); + parts.addAll(partsList.getUnknownPartns()); + } catch (HiveException e) { + // Has to use full name to make sure it does not conflict with org.apache.commons.lang.StringUtils + LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e)); + throw new SemanticException(e.getMessage(), e); + } + SamplePruner samplePruner = parseCtx.getAliasToSamplePruner().get(alias_id); + + for (Partition part : parts) { + // Later the properties have to come from the partition as opposed + // to from the table in order to support versioning. + Path paths[]; + if (samplePruner != null) { + paths = samplePruner.prune(part); + } + else { + paths = part.getPath(); + } + + for (Path p: paths) { + String path = p.toString(); + LOG.debug("Adding " + path + " of table" + alias_id); + // Add the path to alias mapping + if (plan.getPathToAliases().get(path) == null) { + plan.getPathToAliases().put(path, new ArrayList()); + } + plan.getPathToAliases().get(path).add(alias_id); + plan.getPathToPartitionInfo().put(path, Utilities.getPartitionDesc(part)); + LOG.debug("Information added for path " + path); + } + } + plan.getAliasToWork().put(alias_id, topOp); + setKeyAndValueDesc(plan, topOp); + LOG.debug("Created Map Work for " + alias_id); + } + else { + FileSinkOperator fOp = (FileSinkOperator) topOp; + fileSinkDesc fConf = (fileSinkDesc)fOp.getConf(); + // populate local work if needed + } + } + + /** + * set key and value descriptor + * @param plan current plan + * @param topOp current top operator in the path + */ + private static void setKeyAndValueDesc(mapredWork plan, Operator topOp) { + if (topOp instanceof ReduceSinkOperator) { + ReduceSinkOperator rs = (ReduceSinkOperator)topOp; + plan.setKeyDesc(rs.getConf().getKeySerializeInfo()); + int tag = Math.max(0, rs.getConf().getTag()); + List tagToSchema = plan.getTagToValueDesc(); + while (tag + 1 > tagToSchema.size()) { + tagToSchema.add(null); + } + tagToSchema.set(tag, rs.getConf().getValueSerializeInfo()); + } else { + List> children = topOp.getChildOperators(); + if (children != null) { + for(Operator op: children) { + setKeyAndValueDesc(plan, op); + } + } + } + } + + /** + * create a new plan and return + * @return the new plan + */ + public static mapredWork getMapRedWork() { + mapredWork work = new mapredWork(); + work.setPathToAliases(new LinkedHashMap>()); + work.setPathToPartitionInfo(new LinkedHashMap()); + work.setAliasToWork(new HashMap>()); + work.setTagToValueDesc(new ArrayList()); + work.setReducer(null); + return work; + } + + /** + * insert in the map for the operator to row resolver + * @param op operator created + * @param rr row resolver + * @param parseCtx parse context + */ + @SuppressWarnings("nls") + private static Operator putOpInsertMap(Operator op, RowResolver rr, ParseContext parseCtx) + { + OpParseContext ctx = new OpParseContext(rr); + parseCtx.getOpParseCtx().put(op, ctx); + return op; + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMROperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMROperator.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMROperator.java (revision 0) @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer; + +import java.io.Serializable; +import java.util.Map; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.OperatorProcessor; +import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; + +/** + * Processor for the rule - no specific rule fired + */ +public class GenMROperator implements OperatorProcessor { + + public GenMROperator() { + } + + /** + * Reduce Scan encountered + * @param op the reduce sink operator encountered + * @param opProcCtx context + */ + public void process(Operator op, OperatorProcessorContext opProcCtx) throws SemanticException { + GenMRProcContext ctx = (GenMRProcContext)opProcCtx; + + Map, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx(); + mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(), ctx.getCurrAliasId())); + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java (revision 0) @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer; + +import java.io.Serializable; +import java.util.Map; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.OperatorProcessor; +import org.apache.hadoop.hive.ql.exec.TableScanOperator; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.plan.mapredWork; +import org.apache.hadoop.hive.ql.metadata.*; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; + +/** + * Processor for the rule - table scan + */ +public class GenMRTableScan1 implements OperatorProcessor { + public GenMRTableScan1() { + } + + /** + * Table Sink encountered + * @param op the table sink operator encountered + * @param opProcCtx context + */ + public void process(TableScanOperator op, OperatorProcessorContext opProcCtx) throws SemanticException { + GenMRProcContext ctx = (GenMRProcContext)opProcCtx; + ParseContext parseCtx = ctx.getParseCtx(); + Map, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx(); + + // create a dummy task + Task currTask = TaskFactory.get(GenMapRedUtils.getMapRedWork(), parseCtx.getConf()); + Operator currTopOp = op; + ctx.setCurrTask(currTask); + ctx.setCurrTopOp(currTopOp); + + for (String alias : parseCtx.getTopOps().keySet()) { + Operator currOp = parseCtx.getTopOps().get(alias); + if (currOp == op) { + String currAliasId = alias; + ctx.setCurrAliasId(currAliasId); + mapCurrCtx.put(op, new GenMapRedCtx(currTask, currTopOp, currAliasId)); + return; + } + } + assert false; + } + + /** + * Table Sink encountered + * @param op the table sink operator encountered + * @param opProcCtx context + */ + public void process(Operator op, OperatorProcessorContext opProcCtx) throws SemanticException { + // should never be called + assert false; + } +} + Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java (revision 722634) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java (working copy) @@ -18,38 +18,64 @@ package org.apache.hadoop.hive.ql.optimizer; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Stack; + +import org.apache.hadoop.hive.ql.exec.ColumnInfo; +import org.apache.hadoop.hive.ql.exec.ExtractOperator; +import org.apache.hadoop.hive.ql.exec.FileSinkOperator; +import org.apache.hadoop.hive.ql.exec.FilterOperator; +import org.apache.hadoop.hive.ql.exec.GroupByOperator; +import org.apache.hadoop.hive.ql.exec.JoinOperator; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.OperatorFactory; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import org.apache.hadoop.hive.ql.exec.RowSchema; +import org.apache.hadoop.hive.ql.exec.ScriptOperator; +import org.apache.hadoop.hive.ql.exec.SelectOperator; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.parse.DefaultDispatcher; +import org.apache.hadoop.hive.ql.parse.Dispatcher; +import org.apache.hadoop.hive.ql.parse.DefaultOpGraphWalker; +import org.apache.hadoop.hive.ql.parse.OpGraphWalker; +import org.apache.hadoop.hive.ql.parse.OpParseContext; +import org.apache.hadoop.hive.ql.parse.OperatorProcessor; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.QB; -import org.apache.hadoop.hive.ql.parse.OpParseContext; -import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzerFactory; -import org.apache.hadoop.hive.ql.parse.RowResolver; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.OperatorFactory; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.aggregationDesc; +import org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.exprNodeDesc; -import org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.groupByDesc; +import org.apache.hadoop.hive.ql.plan.reduceSinkDesc; import org.apache.hadoop.hive.ql.plan.selectDesc; -import org.apache.hadoop.hive.ql.exec.ColumnInfo; -import org.apache.hadoop.hive.ql.exec.RowSchema; -import java.io.Serializable; -import java.util.List; -import java.util.Iterator; -import java.util.ArrayList; +import org.apache.hadoop.hive.ql.parse.Rule; /** - * Implementation of one of the rule-based optimization steps. ColumnPruner gets the current operator tree. The tree is traversed to find out the columns used - * for all the base tables. If all the columns for a table are not used, a select is pushed on top of that table (to select only those columns). Since this + * Implementation of one of the rule-based optimization steps. ColumnPruner gets the current operator tree. The \ + * tree is traversed to find out the columns used + * for all the base tables. If all the columns for a table are not used, a select is pushed on top of that table + * (to select only those columns). Since this * changes the row resolver, the tree is built again. This can be optimized later to patch the tree. */ public class ColumnPruner implements Transform { - private ParseContext pctx; - + protected ParseContext pGraphContext; + private HashMap, OpParseContext> opToParseCtxMap; + + /** * empty constructor */ public ColumnPruner() { - pctx = null; + pGraphContext = null; } /** @@ -59,7 +85,7 @@ * @return boolean */ private boolean pushSelect(Operator op, List colNames) { - if (pctx.getOpParseCtx().get(op).getRR().getColumnInfos().size() == colNames.size()) return false; + if (pGraphContext.getOpParseCtx().get(op).getRR().getColumnInfos().size() == colNames.size()) return false; return true; } @@ -72,7 +98,7 @@ @SuppressWarnings("nls") private Operator putOpInsertMap(Operator op, RowResolver rr) { OpParseContext ctx = new OpParseContext(rr); - pctx.getOpParseCtx().put(op, ctx); + pGraphContext.getOpParseCtx().put(op, ctx); return op; } @@ -87,7 +113,7 @@ private Operator genSelectPlan(Operator input, List colNames) throws SemanticException { - RowResolver inputRR = pctx.getOpParseCtx().get(input).getRR(); + RowResolver inputRR = pGraphContext.getOpParseCtx().get(input).getRR(); RowResolver outputRR = new RowResolver(); ArrayList col_list = new ArrayList(); @@ -125,26 +151,33 @@ } /** - * Transform the query tree. For each table under consideration, check if all columns are needed. If not, only select the operators needed at - * the beginning and proceed + * Transform the query tree. For each table under consideration, check if all columns are needed. If not, + * only select the operators needed at the beginning and proceed + * @param pactx the current parse context */ public ParseContext transform(ParseContext pactx) throws SemanticException { - this.pctx = pactx; - boolean done = true; - // generate useful columns for all the sources so that they can be pushed immediately after the table scan - for (String alias_id : pctx.getTopOps().keySet()) { - Operator topOp = pctx.getTopOps().get(alias_id); - - // Scan the tree bottom-up and generate columns needed for the top operator - List colNames = topOp.genColLists(pctx.getOpParseCtx()); + this.pGraphContext = pactx; + this.opToParseCtxMap = pGraphContext.getOpParseCtx(); + boolean done = true; + + // generate pruned column list for all relevant operators + ColumnPrunerProcessor cpp = new ColumnPrunerProcessor(opToParseCtxMap); + Dispatcher disp = new DefaultDispatcher(cpp); + OpGraphWalker ogw = new ColumnPrunerWalker(disp); + ogw.startWalking(pGraphContext.getTopOps().values()); + + // create a new select operator if any of input tables' columns can be pruned + for (String alias_id : pGraphContext.getTopOps().keySet()) { + Operator topOp = pGraphContext.getTopOps().get(alias_id); + + List colNames = cpp.getPrunedColList(topOp); // do we need to push a SELECT - all the columns of the table are not used if (pushSelect(topOp, colNames)) { topOp.setChildOperators(null); - // Generate a select and make it a child of the table scan Operator select = genSelectPlan(topOp, colNames); - pctx.getTopSelOps().put(alias_id, select); + pGraphContext.getTopSelOps().put(alias_id, select); done = false; } } @@ -152,17 +185,203 @@ // a select was pushed on top of the table. The old plan is no longer valid. Generate the plan again. // The current tables and the select pushed above (after column pruning) are maintained in the parse context. if (!done) { - SemanticAnalyzer sem = (SemanticAnalyzer)SemanticAnalyzerFactory.get(pctx.getConf(), pctx.getParseTree()); - - resetParseContext(pctx); - sem.init(pctx); - QB qb = new QB(null, null, false); - - sem.doPhase1(pctx.getParseTree(), qb, sem.initPhase1Ctx()); - sem.getMetaData(qb); - sem.genPlan(qb); - pctx = sem.getParseContext(); - } - return pctx; - } + SemanticAnalyzer sem = (SemanticAnalyzer)SemanticAnalyzerFactory.get(pGraphContext.getConf(), pGraphContext.getParseTree()); + + resetParseContext(pGraphContext); + sem.init(pGraphContext); + QB qb = new QB(null, null, false); + + sem.doPhase1(pGraphContext.getParseTree(), qb, sem.initPhase1Ctx()); + sem.getMetaData(qb); + sem.genPlan(qb); + pGraphContext = sem.getParseContext(); + } + return pGraphContext; + } + + /** + * Column pruner processor + **/ + public static class ColumnPrunerProcessor implements OperatorProcessor { + private Map,List> prunedColLists = + new HashMap, List>(); + private HashMap, OpParseContext> opToParseCtxMap; + + public ColumnPrunerProcessor(HashMap, OpParseContext> opToParseContextMap) { + this.opToParseCtxMap = opToParseContextMap; + } + + /** + * @return the prunedColLists + */ + public List getPrunedColList(Operator op) { + return prunedColLists.get(op); + } + + private List genColLists(Operator curOp) throws SemanticException { + List colList = new ArrayList(); + if(curOp.getChildOperators() != null) { + for(Operator child: curOp.getChildOperators()) + colList = Utilities.mergeUniqElems(colList, prunedColLists.get(child)); + } + return colList; + } + + public void process(FilterOperator op, OperatorProcessorContext ctx) throws SemanticException { + exprNodeDesc condn = op.getConf().getPredicate(); + // get list of columns used in the filter + List cl = condn.getCols(); + // merge it with the downstream col list + prunedColLists.put(op, Utilities.mergeUniqElems(genColLists(op), cl)); + } + + public void process(GroupByOperator op, OperatorProcessorContext ctx) throws SemanticException { + List colLists = new ArrayList(); + groupByDesc conf = op.getConf(); + ArrayList keys = conf.getKeys(); + for (exprNodeDesc key : keys) + colLists = Utilities.mergeUniqElems(colLists, key.getCols()); + + ArrayList aggrs = conf.getAggregators(); + for (aggregationDesc aggr : aggrs) { + ArrayList params = aggr.getParameters(); + for (exprNodeDesc param : params) + colLists = Utilities.mergeUniqElems(colLists, param.getCols()); + } + + prunedColLists.put(op, colLists); + } + + public void process(Operator op, OperatorProcessorContext ctx) throws SemanticException { + prunedColLists.put(op, genColLists(op)); + } + + public void process(ReduceSinkOperator op, OperatorProcessorContext ctx) throws SemanticException { + RowResolver redSinkRR = opToParseCtxMap.get(op).getRR(); + reduceSinkDesc conf = op.getConf(); + List> childOperators = op.getChildOperators(); + List> parentOperators = op.getParentOperators(); + List childColLists = new ArrayList(); + + for(Operator child: childOperators) + childColLists = Utilities.mergeUniqElems(childColLists, prunedColLists.get(child)); + + List colLists = new ArrayList(); + ArrayList keys = conf.getKeyCols(); + for (exprNodeDesc key : keys) + colLists = Utilities.mergeUniqElems(colLists, key.getCols()); + + if ((childOperators.size() == 1) && (childOperators.get(0) instanceof JoinOperator)) { + assert parentOperators.size() == 1; + Operator par = parentOperators.get(0); + RowResolver parRR = opToParseCtxMap.get(par).getRR(); + RowResolver childRR = opToParseCtxMap.get(childOperators.get(0)).getRR(); + + for (String childCol : childColLists) { + String [] nm = childRR.reverseLookup(childCol); + ColumnInfo cInfo = redSinkRR.get(nm[0],nm[1]); + if (cInfo != null) { + cInfo = parRR.get(nm[0], nm[1]); + if (!colLists.contains(cInfo.getInternalName())) + colLists.add(cInfo.getInternalName()); + } + } + } + else { + // Reduce Sink contains the columns needed - no need to aggregate from children + ArrayList vals = conf.getValueCols(); + for (exprNodeDesc val : vals) + colLists = Utilities.mergeUniqElems(colLists, val.getCols()); + } + + prunedColLists.put(op, colLists); + } + + public void process(SelectOperator op, OperatorProcessorContext ctx) throws SemanticException { + List cols = new ArrayList(); + + if(op.getChildOperators() != null) { + for(Operator child: op.getChildOperators()) { + // If one of my children is a FileSink or Script, return all columns. + // Without this break, a bug in ReduceSink to Extract edge column pruning will manifest + // which should be fixed before remove this + if ((child instanceof FileSinkOperator) || (child instanceof ScriptOperator)) { + prunedColLists.put(op, getColsFromSelectExpr(op)); + return; + } + cols = Utilities.mergeUniqElems(cols, prunedColLists.get(child)); + } + } + + selectDesc conf = op.getConf(); + if (conf.isSelectStar() && !cols.isEmpty()) { + // The input to the select does not matter. Go over the expressions + // and return the ones which have a marked column + prunedColLists.put(op, getSelectColsFromChildren(op, cols)); + return; + } + prunedColLists.put(op, getColsFromSelectExpr(op)); + } + + private List getColsFromSelectExpr(SelectOperator op) { + List cols = new ArrayList(); + selectDesc conf = op.getConf(); + ArrayList exprList = conf.getColList(); + for (exprNodeDesc expr : exprList) + cols = Utilities.mergeUniqElems(cols, expr.getCols()); + return cols; + } + + private List getSelectColsFromChildren(SelectOperator op, List colList) { + List cols = new ArrayList(); + selectDesc conf = op.getConf(); + ArrayList selectExprs = conf.getColList(); + + for (String col : colList) { + // col is the internal name i.e. position within the expression list + exprNodeDesc expr = selectExprs.get(Integer.parseInt(col)); + cols = Utilities.mergeUniqElems(cols, expr.getCols()); + } + return cols; + } + } + /** + * Walks the op tree in post order fashion (skips selects with file sink or script op children) + */ + public static class ColumnPrunerWalker extends DefaultOpGraphWalker { + + public ColumnPrunerWalker(Dispatcher disp) { + super(disp); + } + + /** + * Walk the given operator + */ + @Override + public void walk(Operator op) throws SemanticException { + boolean walkChildren = true; + + // no need to go further down for a select op with a file sink or script child + // since all cols are needed for these ops + if(op instanceof SelectOperator) { + for(Operator child: op.getChildOperators()) { + if ((child instanceof FileSinkOperator) || (child instanceof ScriptOperator)) + walkChildren = false; + } + } + + if((op.getChildOperators() == null) + || getDispatchedList().containsAll(op.getChildOperators()) + || !walkChildren) { + // all children are done or no need to walk the children + dispatch(op, null); + return; + } + // move all the children to the front of queue + getToWalk().removeAll(op.getChildOperators()); + getToWalk().addAll(0, op.getChildOperators()); + // add self to the end of the queue + getToWalk().add(op); + } + } } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java (revision 0) @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer; + +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Set; +import java.util.Stack; +import java.io.Serializable; +import java.io.File; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import org.apache.hadoop.hive.ql.exec.FileSinkOperator; +import org.apache.hadoop.hive.ql.exec.TableScanOperator; +import org.apache.hadoop.hive.ql.exec.JoinOperator; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.exec.OperatorFactory; +import org.apache.hadoop.hive.ql.plan.mapredWork; +import org.apache.hadoop.hive.ql.plan.reduceSinkDesc; +import org.apache.hadoop.hive.ql.plan.tableDesc; +import org.apache.hadoop.hive.ql.plan.partitionDesc; +import org.apache.hadoop.hive.ql.plan.fileSinkDesc; +import org.apache.hadoop.hive.ql.plan.PlanUtils; +import org.apache.hadoop.hive.ql.metadata.*; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.parse.OperatorProcessor; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; + +/** + * Processor for the rule - table scan followed by reduce sink + */ +public class GenMRRedSink1 implements OperatorProcessor { + + public GenMRRedSink1() { + } + + /** + * Reduce Scan encountered + * @param op the reduce sink operator encountered + * @param opProcCtx context + */ + public void process(ReduceSinkOperator op, OperatorProcessorContext opProcCtx) throws SemanticException { + GenMRProcContext ctx = (GenMRProcContext)opProcCtx; + + Map, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx(); + GenMapRedCtx mapredCtx = mapCurrCtx.get(op.getParentOperators().get(0)); + Task currTask = mapredCtx.getCurrTask(); + mapredWork currPlan = (mapredWork) currTask.getWork(); + Operator currTopOp = mapredCtx.getCurrTopOp(); + String currAliasId = mapredCtx.getCurrAliasId(); + Operator reducer = op.getChildOperators().get(0); + HashMap, Task> opTaskMap = ctx.getOpTaskMap(); + Task opMapTask = opTaskMap.get(reducer); + + ctx.setCurrTopOp(currTopOp); + ctx.setCurrAliasId(currAliasId); + ctx.setCurrTask(currTask); + + // If the plan for this reducer does not exist, initialize the plan + if (opMapTask == null) { + if (currPlan.getReducer() == null) + GenMapRedUtils.initPlan(op, ctx); + else + GenMapRedUtils.splitPlan(op, ctx); + } + // This will happen in case of joins. The current plan can be thrown away after being merged with the + // original plan + else { + GenMapRedUtils.joinPlan(opMapTask, ctx); + currTask = opMapTask; + } + + mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(), ctx.getCurrAliasId())); + } + + /** + * Reduce Scan encountered + * @param op the reduce sink operator encountered + * @param opProcCtx context + */ + public void process(Operator op, OperatorProcessorContext opProcCtx) throws SemanticException { + // should never be called + assert false; + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java (revision 0) @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer; + +import java.io.Serializable; +import java.util.Map; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import org.apache.hadoop.hive.ql.exec.JoinOperator; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.exec.OperatorFactory; +import org.apache.hadoop.hive.ql.plan.mapredWork; +import org.apache.hadoop.hive.ql.plan.reduceSinkDesc; +import org.apache.hadoop.hive.ql.metadata.*; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.parse.OperatorProcessor; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; + +/** + * Processor for the rule - reduce sink followed by reduce sink + */ +public class GenMRRedSink2 implements OperatorProcessor { + + public GenMRRedSink2() { + } + + /** + * Reduce Scan encountered + * @param op the reduce sink operator encountered + * @param opProcCtx context + */ + public void process(ReduceSinkOperator op, OperatorProcessorContext opProcCtx) throws SemanticException { + GenMRProcContext ctx = (GenMRProcContext)opProcCtx; + + Map, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx(); + GenMapRedCtx mapredCtx = mapCurrCtx.get(op.getParentOperators().get(0)); + Task currTask = mapredCtx.getCurrTask(); + Operator currTopOp = mapredCtx.getCurrTopOp(); + String currAliasId = mapredCtx.getCurrAliasId(); + Operator reducer = op.getChildOperators().get(0); + Map, Task> opTaskMap = ctx.getOpTaskMap(); + Task opMapTask = opTaskMap.get(reducer); + + ctx.setCurrTopOp(currTopOp); + ctx.setCurrAliasId(currAliasId); + ctx.setCurrTask(currTask); + + if (opMapTask == null) + GenMapRedUtils.splitPlan(op, ctx); + else { + GenMapRedUtils.joinPlan(opMapTask, ctx); + currTask = opMapTask; + } + + mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(), ctx.getCurrAliasId())); + } + + /** + * Reduce Scan encountered + * @param op the operator encountered + * @param opProcCtx context + */ + public void process(Operator op, OperatorProcessorContext opProcCtx) throws SemanticException { + // should never be called + assert false; + } + +} Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (revision 0) @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer; + +import java.util.List; +import java.util.HashMap; +import java.io.Serializable; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.OperatorProcessor; +import org.apache.hadoop.hive.ql.exec.FileSinkOperator; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.plan.mapredWork; +import org.apache.hadoop.hive.ql.metadata.*; +import org.apache.hadoop.hive.ql.parse.SemanticException; + +/** + * Processor for the rule - table scan followed by reduce sink + */ +public class GenMRFileSink1 implements OperatorProcessor { + + public GenMRFileSink1() { + } + + /** + * File Sink Operator encountered + * @param op the file sink operator encountered + * @param opProcCtx context + */ + public void process(FileSinkOperator op, OperatorProcessorContext opProcCtx) throws SemanticException { + GenMRProcContext ctx = (GenMRProcContext)opProcCtx; + boolean ret = false; + + Task mvTask = ctx.getMvTask(); + Task currTask = ctx.getCurrTask(); + Operator currTopOp = ctx.getCurrTopOp(); + String currAliasId = ctx.getCurrAliasId(); + HashMap, Task> opTaskMap = ctx.getOpTaskMap(); + List> seenOps = ctx.getSeenOps(); + List> rootTasks = ctx.getRootTasks(); + + // Set the move task to be dependent on the current task + if (mvTask != null) + ret = currTask.addDependentTask(mvTask); + + // In case of multi-table insert, the path to alias mapping is needed for all the sources. Since there is no + // reducer, treat it as a plan with null reducer + // If it is a map-only job, the task needs to be processed + if (currTopOp != null) { + Task mapTask = opTaskMap.get(null); + if (mapTask == null) { + assert (!seenOps.contains(currTopOp)); + seenOps.add(currTopOp); + GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, (mapredWork) currTask.getWork(), false, ctx); + opTaskMap.put(null, currTask); + rootTasks.add(currTask); + } + else { + if (!seenOps.contains(currTopOp)) { + seenOps.add(currTopOp); + GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, (mapredWork) mapTask.getWork(), false, ctx); + } + if (ret) + currTask.removeDependentTask(mvTask); + } + } + } + + /** + * @param op the operator encountered + * @param opProcCtx context + */ + public void process(Operator op, OperatorProcessorContext opProcCtx) throws SemanticException { + // should never be called + assert false; + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java (revision 0) @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec; + +import java.io.*; + +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.tableScanDesc; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.conf.Configuration; + +/** + * Table Scan Operator + * If the data is coming from the map-reduce framework, just forward it. + * This will be needed as part of local work when data is not being read as part of map-reduce framework + **/ +public class TableScanOperator extends Operator implements Serializable { + private static final long serialVersionUID = 1L; + public void initialize(Configuration hconf) throws HiveException { + super.initialize(hconf); + // nothing to do really .. + } + + /** + * Currently, the table scan operator does not do anything special other than just forwarding the row. Since the + * table data is always read as part of the map-reduce framework by the mapper. But, this assumption is not true, + * i.e table data is not only read by the mapper, this operator will be enhanced to read the table. + **/ + @Override + public void process(Object row, ObjectInspector rowInspector) + throws HiveException { + forward(row, rowInspector); + } + + /** + * The operator name for this operator type. This is used to construct the rule for an operator + * @return the operator name + **/ + public String getOperatorName() { + return new String("TS"); + } + +} Index: ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java (revision 722634) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java (working copy) @@ -49,6 +49,7 @@ opvec.add(new opTuple (groupByDesc.class, GroupByOperator.class)); opvec.add(new opTuple (joinDesc.class, JoinOperator.class)); opvec.add(new opTuple (limitDesc.class, LimitOperator.class)); + opvec.add(new opTuple (tableScanDesc.class, TableScanOperator.class)); } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (revision 722634) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (working copy) @@ -18,22 +18,20 @@ package org.apache.hadoop.hive.ql.exec; -import java.util.*; -import java.io.*; +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.parse.OpParseContext; +import org.apache.hadoop.hive.ql.plan.explain; import org.apache.hadoop.hive.ql.plan.mapredWork; -import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.conf.Configuration; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hive.ql.plan.explain; -import org.apache.hadoop.hive.ql.parse.RowResolver; -import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.mapred.OutputCollector; /** * Base operator implementation @@ -289,22 +287,12 @@ } } - public List mergeColListsFromChildren(List colList, - HashMap, OpParseContext> opParseCtx) { - return colList; + /** + * returns the name of the operator - specific subclasses can override the name as and when needed + * @return the name of the operator + */ + public String getOperatorName() { + return new String("OP"); } - public List genColLists(HashMap, OpParseContext> opParseCtx) - throws SemanticException { - List colList = new ArrayList(); - if (childOperators != null) - for(Operator o: childOperators) - colList = Utilities.mergeUniqElems(colList, o.genColLists(opParseCtx)); - - List cols = mergeColListsFromChildren(colList, opParseCtx); - OpParseContext ctx = opParseCtx.get(this); - ctx.setColNames(cols); - return cols; - } - } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java (revision 722634) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java (working copy) @@ -18,19 +18,14 @@ package org.apache.hadoop.hive.ql.exec; -import java.io.*; -import java.util.HashMap; -import java.util.List; +import java.io.Serializable; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.hive.ql.parse.OpParseContext; -import org.apache.hadoop.hive.ql.plan.exprNodeDesc; import org.apache.hadoop.hive.ql.plan.filterDesc; import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.ql.parse.RowResolver; +import org.apache.hadoop.io.LongWritable; /** * Filter operator implementation @@ -78,15 +73,4 @@ conditionInspectableObject.o.getClass().getName()); } } - - public List mergeColListsFromChildren(List colList, - HashMap, OpParseContext> opParseCtx) { - exprNodeDesc condn = conf.getPredicate(); - - // get list of columns used in the filter - List cl = condn.getCols(); - - return Utilities.mergeUniqElems(colList, cl); - } - } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (revision 722634) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (working copy) @@ -109,11 +109,17 @@ return parentTasks; } - public void addDependentTask(Task dependent) { + /** + * Add a dependent task on the current task. Return if the dependency already existed or is this a new one + * @return true if the task got added false if it already existed + */ + public boolean addDependentTask(Task dependent) { + boolean ret = false; if (getChildTasks() == null) { setChildTasks(new ArrayList>()); } if (!getChildTasks().contains(dependent)) { + ret = true; getChildTasks().add(dependent); if (dependent.getParentTasks() == null) { dependent.setParentTasks(new ArrayList>()); @@ -122,8 +128,21 @@ dependent.getParentTasks().add(this); } } + return ret; } + /** + * remove the dependent task + * @param dependent the task to remove + */ + public void removeDependentTask(Task dependent) { + if ((getChildTasks() != null) && (getChildTasks().contains(dependent))) { + getChildTasks().remove(dependent); + if ((dependent.getParentTasks() != null) && (dependent.getParentTasks().contains(this))) + dependent.getParentTasks().remove(this); + } + } + public boolean done() { return isdone; } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java (revision 722634) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java (working copy) @@ -18,17 +18,16 @@ package org.apache.hadoop.hive.ql.exec; -import java.io.*; +import java.io.IOException; +import java.io.Serializable; import java.util.ArrayList; -import java.util.List; -import java.util.HashMap; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.exprNodeDesc; import org.apache.hadoop.hive.ql.plan.reduceSinkDesc; import org.apache.hadoop.hive.ql.plan.tableDesc; -import org.apache.hadoop.hive.ql.io.*; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.Serializer; import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject; @@ -38,9 +37,6 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; -import org.apache.hadoop.hive.ql.parse.OpParseContext; -import org.apache.hadoop.hive.ql.parse.RowResolver; -import org.apache.hadoop.hive.ql.parse.SemanticException; /** * Reduce Sink Operator sends output to the reduce stage @@ -200,61 +196,12 @@ throw new HiveException (e); } } - - public List genColLists(HashMap, OpParseContext> opParseCtx) - throws SemanticException { - RowResolver redSinkRR = opParseCtx.get(this).getRR(); - List childColLists = new ArrayList(); - for(Operator o: childOperators) - childColLists = Utilities.mergeUniqElems(childColLists, o.genColLists(opParseCtx)); - - List colLists = new ArrayList(); - ArrayList keys = conf.getKeyCols(); - for (exprNodeDesc key : keys) - colLists = Utilities.mergeUniqElems(colLists, key.getCols()); - - // In case of extract child, see the columns used and propagate them - if ((childOperators.size() == 1) && (childOperators.get(0) instanceof ExtractOperator)) { - assert parentOperators.size() == 1; - Operator par = parentOperators.get(0); - RowResolver parRR = opParseCtx.get(par).getRR(); - - for (String childCol : childColLists) { - String [] nm = redSinkRR.reverseLookup(childCol); - ColumnInfo cInfo = parRR.get(nm[0],nm[1]); - if (!colLists.contains(cInfo.getInternalName())) - colLists.add(cInfo.getInternalName()); - } - } - else if ((childOperators.size() == 1) && (childOperators.get(0) instanceof JoinOperator)) { - assert parentOperators.size() == 1; - Operator par = parentOperators.get(0); - RowResolver parRR = opParseCtx.get(par).getRR(); - RowResolver childRR = opParseCtx.get(childOperators.get(0)).getRR(); - - for (String childCol : childColLists) { - String [] nm = childRR.reverseLookup(childCol); - ColumnInfo cInfo = redSinkRR.get(nm[0],nm[1]); - if (cInfo != null) { - cInfo = parRR.get(nm[0], nm[1]); - if (!colLists.contains(cInfo.getInternalName())) - colLists.add(cInfo.getInternalName()); - } - } - } - else { - - // Reduce Sink contains the columns needed - no need to aggregate from children - ArrayList vals = conf.getValueCols(); - for (exprNodeDesc val : vals) - colLists = Utilities.mergeUniqElems(colLists, val.getCols()); - } - - OpParseContext ctx = opParseCtx.get(this); - ctx.setColNames(colLists); - opParseCtx.put(this, ctx); - return colLists; + /** + * @return the name of the operator + */ + public String getOperatorName() { + return new String("RS"); } - + } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (revision 722634) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (working copy) @@ -163,4 +163,11 @@ throw new HiveException (e); } } + + /** + * @return the name of the operator + */ + public String getOperatorName() { + return new String("FS"); + } } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java (revision 722634) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java (working copy) @@ -18,20 +18,16 @@ package org.apache.hadoop.hive.ql.exec; -import java.io.*; +import java.io.Serializable; import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.parse.OpParseContext; import org.apache.hadoop.hive.ql.plan.exprNodeDesc; import org.apache.hadoop.hive.ql.plan.selectDesc; import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.ql.parse.SemanticException; /** * Select operator implementation @@ -91,61 +87,4 @@ } forward(output, outputObjectInspector); } - - private List getColsFromExpr(HashMap, OpParseContext> opParseCtx) { - List cols = new ArrayList(); - ArrayList exprList = conf.getColList(); - for (exprNodeDesc expr : exprList) - cols = Utilities.mergeUniqElems(cols, expr.getCols()); - List listExprs = new ArrayList(); - for (int pos = 0; pos < exprList.size(); pos++) - listExprs.add(new Integer(pos)); - OpParseContext ctx = opParseCtx.get(this); - ctx.setColNames(cols); - opParseCtx.put(this, ctx); - return cols; - } - - private List getColsFromExpr(List colList, - HashMap, OpParseContext> opParseCtx) { - if (colList.isEmpty()) - return getColsFromExpr(opParseCtx); - - List cols = new ArrayList(); - ArrayList selectExprs = conf.getColList(); - List listExprs = new ArrayList(); - - for (String col : colList) { - // col is the internal name i.e. position within the expression list - Integer pos = new Integer(col); - exprNodeDesc expr = selectExprs.get(pos.intValue()); - cols = Utilities.mergeUniqElems(cols, expr.getCols()); - listExprs.add(pos); - } - - OpParseContext ctx = opParseCtx.get(this); - ctx.setColNames(cols); - opParseCtx.put(this, ctx); - return cols; - } - - public List genColLists(HashMap, OpParseContext> opParseCtx) - throws SemanticException { - List cols = new ArrayList(); - - for(Operator o: childOperators) { - // if one of my children is a fileSink, return everything - if ((o instanceof FileSinkOperator) || (o instanceof ScriptOperator)) - return getColsFromExpr(opParseCtx); - - cols = Utilities.mergeUniqElems(cols, o.genColLists(opParseCtx)); - } - - if (conf.isSelectStar()) - // The input to the select does not matter. Go over the expressions and return the ones which have a marked column - return getColsFromExpr(cols, opParseCtx); - - return getColsFromExpr(opParseCtx); - } - } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/OpGraphWalker.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/OpGraphWalker.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/OpGraphWalker.java (revision 0) @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.io.Serializable; +import java.util.Collection; + +import org.apache.hadoop.hive.ql.exec.Operator; + +/** + * Interface for operator graph walker. + */ +public interface OpGraphWalker { + + /** + * starting point for walking. + * + * @param startOps list of starting operators + * @throws SemanticException + */ + public abstract void startWalking(Collection> startOps) + throws SemanticException; + +} \ No newline at end of file Property changes on: ql/src/java/org/apache/hadoop/hive/ql/parse/OpGraphWalker.java ___________________________________________________________________ Added: svn:executable + * Index: ql/src/java/org/apache/hadoop/hive/ql/parse/GenMapRedWalker.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenMapRedWalker.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenMapRedWalker.java (revision 0) @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.io.Serializable; +import java.util.List; +import java.util.Stack; +import java.util.regex.Pattern; +import java.util.regex.Matcher; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import org.apache.hadoop.hive.ql.exec.*; + +/** + * Walks the operator tree in pre order fashion + */ +public class GenMapRedWalker extends DefaultOpGraphWalker { + private Stack> opStack; + + /** + * constructor of the walker - the dispatcher is passed + * @param disp the dispatcher to be called for each node visited + */ + public GenMapRedWalker(Dispatcher disp) { + super(disp); + opStack = new Stack>(); + } + + /** + * Walk the given operator + * @param op operator being walked + */ + @Override + public void walk(Operator op) throws SemanticException { + List> children = op.getChildOperators(); + + // maintain the stack of operators encountered + opStack.push(op); + dispatch(op, opStack); + + // kids of reduce sink operator need not be traversed again + if ((children == null) || + ((op instanceof ReduceSinkOperator) && (getDispatchedList().containsAll(children)))) { + opStack.pop(); + return; + } + + // move all the children to the front of queue + for (Operator ch : children) + walk(ch); + + // done with this operator + opStack.pop(); + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java (revision 722634) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java (working copy) @@ -40,7 +40,20 @@ private HashMap destToSelExpr; private HashMap destToWhereExpr; private HashMap destToGroupby; + /** + * ClusterBy is a short name for both DistributeBy and SortBy. + */ private HashMap destToClusterby; + /** + * DistributeBy controls the hashcode of the row, which determines which reducer + * the rows will go to. + */ + private HashMap destToDistributeby; + /** + * SortBy controls the reduce keys, which affects the order of rows + * that the reducer receives. + */ + private HashMap destToSortby; private HashMap destToLimit; private int outerQueryLimit; @@ -59,6 +72,8 @@ this.destToWhereExpr = new HashMap(); this.destToGroupby = new HashMap(); this.destToClusterby = new HashMap(); + this.destToDistributeby = new HashMap(); + this.destToSortby = new HashMap(); this.destToLimit = new HashMap(); this.destToAggregationExprs = new HashMap >(); @@ -101,10 +116,33 @@ this.nameToDest.put(clause, ast); } + /** + * Set the Cluster By AST for the clause. + * @param clause the name of the clause + * @param ast the abstract syntax tree + */ public void setClusterByExprForClause(String clause, CommonTree ast) { this.destToClusterby.put(clause, ast); } + /** + * Set the Distribute By AST for the clause. + * @param clause the name of the clause + * @param ast the abstract syntax tree + */ + public void setDistributeByExprForClause(String clause, CommonTree ast) { + this.destToDistributeby.put(clause, ast); + } + + /** + * Set the Sort By AST for the clause. + * @param clause the name of the clause + * @param ast the abstract syntax tree + */ + public void setSortByExprForClause(String clause, CommonTree ast) { + this.destToSortby.put(clause, ast); + } + public void setSrcForAlias(String alias, CommonTree ast) { this.aliasToSrc.put(alias.toLowerCase(), ast); } @@ -137,10 +175,33 @@ return this.destToSelExpr.get(clause); } + /** + * Get the Cluster By AST for the clause. + * @param clause the name of the clause + * @return the abstract syntax tree + */ public CommonTree getClusterByForClause(String clause) { return this.destToClusterby.get(clause); } + /** + * Get the Distribute By AST for the clause. + * @param clause the name of the clause + * @return the abstract syntax tree + */ + public CommonTree getDistributeByForClause(String clause) { + return this.destToDistributeby.get(clause); + } + + /** + * Get the Sort By AST for the clause. + * @param clause the name of the clause + * @return the abstract syntax tree + */ + public CommonTree getSortByForClause(String clause) { + return this.destToSortby.get(clause); + } + public CommonTree getSrcForAlias(String alias) { return this.aliasToSrc.get(alias.toLowerCase()); } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/PrintOpTreeProcessor.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/PrintOpTreeProcessor.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/PrintOpTreeProcessor.java (revision 0) @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.io.PrintStream; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Stack; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext; + +public class PrintOpTreeProcessor implements OperatorProcessor { + + private PrintStream out; + private HashMap, Integer> opMap = new HashMap, Integer>(); + private Integer curNum = 0; + + public PrintOpTreeProcessor() { + out = System.out; + } + + public PrintOpTreeProcessor(PrintStream o) { + out = o; + } + + private String getParents(Operator op) { + StringBuilder ret = new StringBuilder("["); + boolean first = true; + if(op.getParentOperators() != null) { + for(Operator parent : op.getParentOperators()) { + if(!first) + ret.append(","); + ret.append(opMap.get(parent)); + first = false; + } + } + ret.append("]"); + return ret.toString(); + } + + private String getChildren(Operator op) { + StringBuilder ret = new StringBuilder("["); + boolean first = true; + if(op.getChildOperators() != null) { + for(Operator child : op.getChildOperators()) { + if(!first) + ret.append(","); + ret.append(opMap.get(child)); + first = false; + } + } + ret.append("]"); + return ret.toString(); + } + + public void process(Operator op, OperatorProcessorContext ctx) throws SemanticException { + if (opMap.get(op) == null) { + opMap.put(op, curNum++); + } + out.println("[" + opMap.get(op) + "] " + op.getClass().getName() + " =p=> " + getParents(op) + " =c=> " + getChildren(op)); + if(op.getConf() == null) { + return; + } + } +} Property changes on: ql/src/java/org/apache/hadoop/hive/ql/parse/PrintOpTreeProcessor.java ___________________________________________________________________ Added: svn:executable + * Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultOpGraphWalker.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultOpGraphWalker.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultOpGraphWalker.java (revision 0) @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.Stack; + +import org.apache.hadoop.hive.ql.exec.Operator; + +/** + * base class for operator graph walker + * this class takes list of starting ops and walks them one by one. it maintains list of walked + * operators (dispatchedList) and a list of operators that are discovered but not yet dispatched + */ +public abstract class DefaultOpGraphWalker implements OpGraphWalker { + + List> toWalk = new ArrayList>(); + Set> dispatchedList = new HashSet>(); + Dispatcher dispatcher; + + /** + * Constructor + * @param ctx graph of operators to walk + * @param disp dispatcher to call for each op encountered + */ + public DefaultOpGraphWalker(Dispatcher disp) { + this.dispatcher = disp; + } + + /** + * @return the toWalk + */ + public List> getToWalk() { + return toWalk; + } + + /** + * @return the doneList + */ + public Set> getDispatchedList() { + return dispatchedList; + } + + /** + * Dispatch the current operator + * @param op operator being walked + * @param opStack stack of operators encountered + * @throws SemanticException + */ + public void dispatch(Operator op, Stack opStack) throws SemanticException { + this.dispatcher.dispatch(op, opStack); + this.dispatchedList.add(op); + } + + /** + * starting point for walking + * @throws SemanticException + */ + public void startWalking(Collection> startOps) throws SemanticException { + toWalk.addAll(startOps); + while(toWalk.size() > 0) + walk(toWalk.remove(0)); + } + + /** + * walk the current operator and its descendants + * @param op current operator in the graph + * @throws SemanticException + */ + public abstract void walk(Operator op) throws SemanticException; +} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/TopoWalker.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/TopoWalker.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TopoWalker.java (revision 0) @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.io.Serializable; + +import org.apache.hadoop.hive.ql.exec.Operator; + +/** + * walks the tree in toplogically sorted order + */ +public class TopoWalker extends DefaultOpGraphWalker { + + public TopoWalker(Dispatcher disp) { + super(disp); + } + + @Override + public void walk(Operator op) throws SemanticException { + // the dispatcher does not care about the stack - so dont maintin it + dispatch(op, null); + if(op.getChildOperators() != null) { + for(Operator child : op.getChildOperators()) { + if(getDispatchedList().containsAll(child.getParentOperators())) { + walk(child); + } + } + } + } + +} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultDispatcher.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultDispatcher.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultDispatcher.java (revision 0) @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.io.Serializable; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.List; +import java.util.Stack; +import java.lang.ClassNotFoundException; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.OperatorFactory; +import org.apache.hadoop.hive.ql.exec.OperatorFactory.opTuple; + +/** + * Dispatches calls to relevant method in processor + */ +public class DefaultDispatcher implements Dispatcher { + + private OperatorProcessor opProcessor; + + /** + * constructor + * @param opp operator processor that handles actual processing of the node + */ + public DefaultDispatcher(OperatorProcessor opp) { + this.opProcessor = opp; + } + + /** + * dispatcher function + * @param op operator to process + * @param opStack the operators encountered so far + * @throws SemanticException + */ + public void dispatch(Operator op, Stack> opStack) + throws SemanticException { + + // If the processor has registered a process method for the particular operator, invoke it. + // Otherwise implement the generic function, which would definitely be implemented + for(opTuple opt : OperatorFactory.opvec) { + if(opt.opClass.isInstance(op)) { + Method pcall; + try { + pcall = opProcessor.getClass().getMethod("process", opt.opClass, + Class.forName("org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext")); + pcall.invoke(opProcessor, op, null); + return; + } catch (SecurityException e) { + assert false; + } catch (NoSuchMethodException e) { + assert false; + } catch (IllegalArgumentException e) { + assert false; + } catch (IllegalAccessException e) { + assert false; + } catch (InvocationTargetException e) { + throw new SemanticException(e.getTargetException()); + } catch (ClassNotFoundException e) { + assert false; + } + } + } + + try { + // no method found - invoke the generic function + Method pcall = opProcessor.getClass().getMethod("process", Class.forName("org.apache.hadoop.hive.ql.exec.Operator"), + Class.forName("org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext")); + + pcall.invoke(opProcessor, ((Operator)op), null); + return; + } catch (SecurityException e) { + assert false; + } catch (NoSuchMethodException e) { + assert false; + } catch (IllegalArgumentException e) { + assert false; + } catch (IllegalAccessException e) { + assert false; + } catch (InvocationTargetException e) { + throw new SemanticException(e.getTargetException()); + } catch (ClassNotFoundException e) { + assert false; + } + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (revision 722634) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (working copy) @@ -54,6 +54,8 @@ TOK_GROUPBY; TOK_ORDERBY; TOK_CLUSTERBY; +TOK_DISTRIBUTEBY; +TOK_SORTBY; TOK_UNION; TOK_JOIN; TOK_LEFTOUTERJOIN; @@ -391,16 +393,24 @@ whereClause? groupByClause? orderByClause? - clusterByClause? - limitClause? -> ^(TOK_QUERY fromClause ^(TOK_INSERT insertClause selectClause whereClause? groupByClause? orderByClause? clusterByClause? limitClause?)) + clusterByClause? + distributeByClause? + sortByClause? + limitClause? -> ^(TOK_QUERY fromClause ^(TOK_INSERT insertClause + selectClause whereClause? groupByClause? orderByClause? clusterByClause? + distributeByClause? sortByClause? limitClause?)) | selectClause fromClause whereClause? groupByClause? orderByClause? - clusterByClause? - limitClause? -> ^(TOK_QUERY fromClause ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) selectClause whereClause? groupByClause? orderByClause? clusterByClause? limitClause?)) + clusterByClause? + distributeByClause? + sortByClause? + limitClause? -> ^(TOK_QUERY fromClause ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) + selectClause whereClause? groupByClause? orderByClause? clusterByClause? + distributeByClause? sortByClause? limitClause?)) ; @@ -410,20 +420,28 @@ selectClause whereClause? groupByClause? - orderByClause? - clusterByClause? - limitClause? -> ^(TOK_INSERT insertClause? selectClause whereClause? groupByClause? orderByClause? clusterByClause? limitClause?) + orderByClause? + clusterByClause? + distributeByClause? + sortByClause? + limitClause? -> ^(TOK_INSERT insertClause? + selectClause whereClause? groupByClause? orderByClause? clusterByClause? + distributeByClause? sortByClause? limitClause?) | selectClause whereClause? groupByClause? - orderByClause? - clusterByClause? - limitClause? -> ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) selectClause whereClause? groupByClause? orderByClause? clusterByClause? limitClause?) + orderByClause? + clusterByClause? + distributeByClause? + sortByClause? + limitClause? -> ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) + selectClause whereClause? groupByClause? orderByClause? clusterByClause? + distributeByClause? sortByClause? limitClause?) ; - + insertClause - : + : KW_INSERT KW_OVERWRITE destination -> ^(TOK_DESTINATION destination) ; @@ -446,12 +464,17 @@ KW_SELECT (KW_ALL | dist=KW_DISTINCT)? selectList -> {$dist == null}? ^(TOK_SELECT selectList) -> ^(TOK_SELECTDI selectList) + | + KW_SELECT KW_TRANSFORM trfmClause -> ^(TOK_SELECT ^(TOK_SELEXPR trfmClause) ) + | + KW_MAP trfmClause -> ^(TOK_SELECT ^(TOK_SELEXPR trfmClause) ) + | + KW_REDUCE trfmClause -> ^(TOK_SELECT ^(TOK_SELEXPR trfmClause) ) ; selectList : selectItem ( COMMA selectItem )* -> selectItem+ - | trfmClause -> ^(TOK_SELEXPR trfmClause) ; selectItem @@ -461,10 +484,9 @@ trfmClause : - KW_TRANSFORM - LPAREN expressionList RPAREN + ( LPAREN expressionList RPAREN | expressionList ) KW_USING StringLiteral - (KW_AS LPAREN aliasList RPAREN)? + (KW_AS (LPAREN aliasList RPAREN | aliasList) )? -> ^(TOK_TRANSFORM expressionList StringLiteral aliasList?) ; @@ -585,18 +607,25 @@ clusterByClause : - KW_CLUSTER KW_BY - Identifier + KW_CLUSTER KW_BY + Identifier ( COMMA Identifier )* -> ^(TOK_CLUSTERBY Identifier+) ; -clusterByExpression - : - expression +distributeByClause: + KW_DISTRIBUTE KW_BY + Identifier + ( COMMA Identifier )* -> ^(TOK_DISTRIBUTEBY Identifier+) ; +sortByClause: + KW_SORT KW_BY + columnNameOrder + ( COMMA columnNameOrder)* -> ^(TOK_SORTBY columnNameOrder+) + ; + // fun(par1, par2, par3) -function +function : // LEFT and RIGHT keywords are also function names Identifier LPAREN ( @@ -822,6 +851,8 @@ KW_TRANSFORM : 'TRANSFORM'; KW_USING: 'USING'; KW_CLUSTER: 'CLUSTER'; +KW_DISTRIBUTE: 'DISTRIBUTE'; +KW_SORT: 'SORT'; KW_UNION: 'UNION'; KW_LOAD: 'LOAD'; KW_DATA: 'DATA'; @@ -849,6 +880,7 @@ KW_STRING: 'STRING'; KW_ARRAY: 'ARRAY'; KW_MAP: 'MAP'; +KW_REDUCE: 'REDUCE'; KW_PARTITIONED: 'PARTITIONED'; KW_CLUSTERED: 'CLUSTERED'; KW_SORTED: 'SORTED'; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (revision 722634) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (working copy) @@ -63,6 +63,8 @@ INVALID_TBL_DDL_SERDE("Either list of columns or a custom serializer should be specified"), TARGET_TABLE_COLUMN_MISMATCH("Cannot insert into target table because column number/types are different"), TABLE_ALIAS_NOT_ALLOWED("Table Alias not Allowed in Sampling Clause"), + CLUSTERBY_DISTRIBUTEBY_CONFLICT("Cannot have both Cluster By and Distribute By Clauses"), + CLUSTERBY_SORTBY_CONFLICT("Cannot have both Cluster By and Sort By Clauses"), NON_BUCKETED_TABLE("Sampling Expression Needed for Non-Bucketed Table"); private String mesg; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/Rule.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/Rule.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/Rule.java (revision 0) @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.util.Stack; +import java.io.Serializable; +import org.apache.hadoop.hive.ql.exec.Operator; + +/** + * Rule interface for Operators + * Used in operator dispatching to dispatch process/visitor functions for operators + */ +public interface Rule { + + /** + * @return the cost of the rule - the lower the cost, the better the rule matches + * @throws SemanticException + */ + public int cost(Stack> stack) throws SemanticException; + + /** + * @return the name of the rule - may be useful for debugging + */ + public String getName(); +} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 722634) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -37,6 +37,13 @@ import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.metadata.*; import org.apache.hadoop.hive.ql.optimizer.Optimizer; +import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext; +import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; +import org.apache.hadoop.hive.ql.optimizer.GenMROperator; +import org.apache.hadoop.hive.ql.optimizer.GenMRTableScan1; +import org.apache.hadoop.hive.ql.optimizer.GenMRFileSink1; +import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink1; +import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink2; import org.apache.hadoop.hive.ql.plan.*; import org.apache.hadoop.hive.ql.typeinfo.TypeInfo; import org.apache.hadoop.hive.ql.typeinfo.TypeInfoFactory; @@ -366,6 +373,26 @@ } break; + case HiveParser.TOK_DISTRIBUTEBY: { + // Get the distribute by aliases - these are aliased to the entries in the + // select list + qbp.setDistributeByExprForClause(ctx_1.dest, ast); + if (qbp.getClusterByForClause(ctx_1.dest) != null) { + throw new SemanticException(ErrorMsg.CLUSTERBY_DISTRIBUTEBY_CONFLICT.getMsg(ast)); + } + } + break; + + case HiveParser.TOK_SORTBY: { + // Get the sort by aliases - these are aliased to the entries in the + // select list + qbp.setSortByExprForClause(ctx_1.dest, ast); + if (qbp.getClusterByForClause(ctx_1.dest) != null) { + throw new SemanticException(ErrorMsg.CLUSTERBY_SORTBY_CONFLICT.getMsg(ast)); + } + } + break; + case HiveParser.TOK_GROUPBY: { // Get the groupby aliases - these are aliased to the entries in the // select list @@ -824,7 +851,8 @@ } @SuppressWarnings("nls") - private Operator putOpInsertMap(Operator op, RowResolver rr) { + private Operator putOpInsertMap(Operator op, RowResolver rr) + { OpParseContext ctx = new OpParseContext(rr); opParseCtx.put(op, ctx); return op; @@ -2011,37 +2039,74 @@ private Operator genReduceSinkPlan(String dest, QB qb, Operator input, int numReducers) throws SemanticException { - // First generate the expression for the key - // The cluster by clause has the aliases for the keys - ArrayList keyCols = new ArrayList(); RowResolver inputRR = opParseCtx.get(input).getRR(); - - CommonTree clby = qb.getParseInfo().getClusterByForClause(dest); - if (clby != null) { - int ccount = clby.getChildCount(); + + // First generate the expression for the partition and sort keys + // The cluster by clause / distribute by clause has the aliases for partition function + CommonTree partitionExprs = qb.getParseInfo().getClusterByForClause(dest); + if (partitionExprs == null) { + partitionExprs = qb.getParseInfo().getDistributeByForClause(dest); + } + ArrayList partitionCols = new ArrayList(); + if (partitionExprs != null) { + int ccount = partitionExprs.getChildCount(); for(int i=0; i valueCols = new ArrayList(); + CommonTree sortExprs = qb.getParseInfo().getClusterByForClause(dest); + if (sortExprs == null) { + sortExprs = qb.getParseInfo().getSortByForClause(dest); + } + ArrayList sortCols = new ArrayList(); + StringBuilder order = new StringBuilder(); + if (sortExprs != null) { + int ccount = sortExprs.getChildCount(); + for(int i=0; i valueCols = new ArrayList(); for(ColumnInfo colInfo: inputRR.getColumnInfos()) { valueCols.add(new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName())); } Operator interim = putOpInsertMap( OperatorFactory.getAndMakeChild( - PlanUtils.getReduceSinkDesc(keyCols, valueCols, -1, keyCols.size(), numReducers, false), + PlanUtils.getReduceSinkDesc(sortCols, valueCols, -1, partitionCols, order.toString(), + numReducers, false), new RowSchema(inputRR.getColumnInfos()), input), inputRR); @@ -2227,8 +2292,12 @@ for (int i=0; i, Task> opTaskMap = - new HashMap, Task>(); - for (String alias_id : this.topOps.keySet()) { - Operator topOp = this.topOps.get(alias_id); - Operator reduceSink = getReduceSink(topOp); - Operator reducer = null; - if (reduceSink != null) - reducer = reduceSink.getChildOperators().get(0); - Task rootTask = opTaskMap.get(reducer); - if (rootTask == null) { - rootTask = TaskFactory.get(getMapRedWork(), this.conf); - opTaskMap.put(reducer, rootTask); - ((mapredWork) rootTask.getWork()).setReducer(reducer); - reduceSinkDesc desc = (reduceSink == null) ? null : (reduceSinkDesc)reduceSink.getConf(); + // generate map reduce plans + GenMRProcContext procCtx = + new GenMRProcContext( + new HashMap, Task>(), + new ArrayList>(), + getParseContext(), mvTask, this.rootTasks, this.scratchDir, this.randomid, this.pathid, + new HashMap, GenMapRedCtx>()); - // The number of reducers may be specified in the plan in some cases, or may need to be inferred - if (desc != null) { - if (desc.getNumReducers() != -1) - ((mapredWork) rootTask.getWork()).setNumReduceTasks(new Integer(desc.getNumReducers())); - else if (desc.getInferNumReducers() == true) - ((mapredWork) rootTask.getWork()).setInferNumReducers(true); - } - this.rootTasks.add(rootTask); - } - genTaskPlan(topOp, rootTask, opTaskMap, mvTask); + // create a walker which walks the tree in a DFS manner while maintaining the operator stack. The dispatcher + // generates the plan from the operator tree + Map opRules = new LinkedHashMap(); + opRules.put(new RuleRegExp(new String("R0"), ".*"), new GenMROperator()); + opRules.put(new RuleRegExp(new String("R1"), "TS"), new GenMRTableScan1()); + opRules.put(new RuleRegExp(new String("R2"), "TS.*RS"), new GenMRRedSink1()); + opRules.put(new RuleRegExp(new String("R3"), "RS.*RS"), new GenMRRedSink2()); + opRules.put(new RuleRegExp(new String("R4"), ".*FS"), new GenMRFileSink1()); - // Generate the map work for this alias_id - PartitionPruner pruner = this.aliasToPruner.get(alias_id); - Set parts = null; - try { - // pass both confirmed and unknown partitions through the map-reduce framework - PartitionPruner.PrunedPartitionList partsList = pruner.prune(); - parts = partsList.getConfirmedPartns(); - parts.addAll(partsList.getUnknownPartns()); - } catch (HiveException e) { - // Has to use full name to make sure it does not conflict with org.apache.commons.lang.StringUtils - LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e)); - throw new SemanticException(e.getMessage(), e); - } - SamplePruner samplePruner = this.aliasToSamplePruner.get(alias_id); - mapredWork plan = (mapredWork) rootTask.getWork(); - for (Partition part : parts) { - // Later the properties have to come from the partition as opposed - // to from the table in order to support versioning. - Path paths[]; - if (samplePruner != null) { - paths = samplePruner.prune(part); - } - else { - paths = part.getPath(); - } - for (Path p: paths) { - String path = p.toString(); - LOG.debug("Adding " + path + " of table" + alias_id); - // Add the path to alias mapping - if (plan.getPathToAliases().get(path) == null) { - plan.getPathToAliases().put(path, new ArrayList()); - } - plan.getPathToAliases().get(path).add(alias_id); - plan.getPathToPartitionInfo().put(path, Utilities.getPartitionDesc(part)); - LOG.debug("Information added for path " + path); - } - } - plan.getAliasToWork().put(alias_id, topOp); - setKeyAndValueDesc(plan, topOp); - LOG.debug("Created Map Work for " + alias_id); - } - } + // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along + Dispatcher disp = new DefaultRuleDispatcher(opRules, procCtx); + + OpGraphWalker ogw = new GenMapRedWalker(disp); + ogw.startWalking(this.topOps.values()); - private void setKeyAndValueDesc(mapredWork plan, Operator topOp) { - if (topOp instanceof ReduceSinkOperator) { - ReduceSinkOperator rs = (ReduceSinkOperator)topOp; - plan.setKeyDesc(rs.getConf().getKeySerializeInfo()); - int tag = Math.max(0, rs.getConf().getTag()); - List tagToSchema = plan.getTagToValueDesc(); - while (tag + 1 > tagToSchema.size()) { - tagToSchema.add(null); - } - tagToSchema.set(tag, rs.getConf().getValueSerializeInfo()); - } else { - List> children = topOp.getChildOperators(); - if (children != null) { - for(Operator op: children) { - setKeyAndValueDesc(plan, op); - } - } - } + // reduce sink does not have any kids + breakOperatorTree(procCtx.getRootOps()); } - @SuppressWarnings("nls") - private void genTaskPlan(Operator op, Task currTask, - HashMap, Task> redTaskMap, - Task mvTask) { - // Check if this is a file sink operator - if ((op.getClass() == FileSinkOperator.class) && (mvTask != null)) { - // If this is a file sink operator then set the move task to be dependent - // on the current task - currTask.addDependentTask(mvTask); - } - List> childOps = op.getChildOperators(); - - // If there are no children then we are done - if (childOps == null) { + private void breakOperatorTree(Collection> topOps) { + if (topOps == null) return; + Iterator> topOpsIter = topOps.iterator(); + while (topOpsIter.hasNext()) { + Operator topOp = topOpsIter.next(); + breakOperatorTree(topOp); } + } - // Otherwise go through the children and check for the operator following - // the reduce sink operator - mapredWork plan = (mapredWork) currTask.getWork(); - for (int i = 0; i < childOps.size(); ++i) { - Operator child = childOps.get(i); - - if (child.getClass() == ReduceSinkOperator.class) { - // Get the operator following the reduce sink - assert (child.getChildOperators().size() == 1); - - Operator reducer = child.getChildOperators().get(0); - assert (plan.getReducer() != null); - if (plan.getReducer() == reducer) { - if (child.getChildOperators().get(0).getClass() == JoinOperator.class) - plan.setNeedsTagging(true); - - // Recurse on the reducer - genTaskPlan(reducer, currTask, redTaskMap, mvTask); - } - else if (plan.getReducer() != reducer) { - Task ctask = null; - mapredWork cplan = null; - - // First check if the reducer already has an associated task - ctask = redTaskMap.get(reducer); - if (ctask == null) { - // For this case we need to generate a new task - cplan = getMapRedWork(); - ctask = TaskFactory.get(cplan, this.conf); - // Add the reducer - cplan.setReducer(reducer); - if (((reduceSinkDesc)child.getConf()).getNumReducers() != -1) - cplan.setNumReduceTasks(new Integer(((reduceSinkDesc)child.getConf()).getNumReducers())); - else - cplan.setInferNumReducers(((reduceSinkDesc)child.getConf()).getInferNumReducers()); - redTaskMap.put(reducer, ctask); - - // Recurse on the reducer - genTaskPlan(reducer, ctask, redTaskMap, mvTask); - - // generate the temporary file - String taskTmpDir = this.scratchDir + File.separator + this.randomid + '.' + this.pathid ; - this.pathid++; - - // Go over the row schema of the input operator and generate the - // column names using that - StringBuilder sb = new StringBuilder(); - boolean isfirst = true; - for(ColumnInfo colInfo: op.getSchema().getSignature()) { - if (!isfirst) { - sb.append(","); - } - sb.append(colInfo.getInternalName()); - isfirst = false; - } - - tableDesc tt_desc = PlanUtils.getBinaryTableDesc( - PlanUtils.getFieldSchemasFromRowSchema(op.getSchema(), "temporarycol")); - - // Create a file sink operator for this file name - Operator fs_op = putOpInsertMap(OperatorFactory.get(new fileSinkDesc(taskTmpDir, tt_desc), - op.getSchema()), null); - - // replace the reduce child with this operator - childOps.set(i, fs_op); - - List> parent = new ArrayList>(); - parent.add(op); - fs_op.setParentOperators(parent); - - // Add the path to alias mapping - if (cplan.getPathToAliases().get(taskTmpDir) == null) { - cplan.getPathToAliases().put(taskTmpDir, new ArrayList()); - } - - String streamDesc; - if (child.getChildOperators().get(0).getClass() == JoinOperator.class) - streamDesc = "$INTNAME"; - else - streamDesc = taskTmpDir; - - - cplan.getPathToAliases().get(taskTmpDir).add(streamDesc); - - cplan.getPathToPartitionInfo().put(taskTmpDir, - new partitionDesc(tt_desc, null)); - - cplan.getAliasToWork().put(streamDesc, child); - setKeyAndValueDesc(cplan, child); - - // Make this task dependent on the current task - currTask.addDependentTask(ctask); - - // TODO: Allocate work to remove the temporary files and make that - // dependent on the cTask - if (child.getChildOperators().get(0).getClass() == JoinOperator.class) - cplan.setNeedsTagging(true); - } - } - child.setChildOperators(null); - - } else { - // For any other operator just recurse - genTaskPlan(child, currTask, redTaskMap, mvTask); - } - } + private void breakOperatorTree(Operator topOp) { + breakOperatorTree(topOp.getChildOperators()); + if (topOp instanceof ReduceSinkOperator) + topOp.setChildOperators(null); } + @SuppressWarnings("nls") public Phase1Ctx initPhase1Ctx() { @@ -3158,23 +3056,6 @@ return ctx_1; } - private mapredWork getMapRedWork() { - - mapredWork work = new mapredWork(); - work.setPathToAliases(new LinkedHashMap>()); - work.setPathToPartitionInfo(new LinkedHashMap()); - work.setAliasToWork(new HashMap>()); - work.setTagToValueDesc(new ArrayList()); - work.setReducer(null); - - return work; - } - - private boolean pushSelect(Operator op, List colNames) { - if (opParseCtx.get(op).getRR().getColumnInfos().size() == colNames.size()) return false; - return true; - } - @Override @SuppressWarnings("nls") public void analyzeInternal(CommonTree ast, Context ctx) throws SemanticException { @@ -3203,7 +3084,7 @@ pCtx = optm.optimize(); init(pCtx); qb = pCtx.getQB(); - + // Do any partition pruning genPartitionPruners(qb); LOG.info("Completed partition pruning"); Index: ql/src/java/org/apache/hadoop/hive/ql/parse/OpParseContext.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/OpParseContext.java (revision 722634) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/OpParseContext.java (working copy) @@ -18,20 +18,16 @@ package org.apache.hadoop.hive.ql.parse; -import java.util.List; /** * Implementation of the Operator Parse Context. It maintains the parse context * that may be needed by an operator. Currently, it only maintains the row - * resolver and the list of columns used by the operator + * resolver. **/ public class OpParseContext { private RowResolver rr; // row resolver for the operator - // list of internal column names used - private List colNames; - /** * @param rr row resolver */ @@ -52,18 +48,4 @@ public void setRR(RowResolver rr) { this.rr = rr; } - - /** - * @return the column names desired - */ - public List getColNames() { - return colNames; - } - - /** - * @param colNames the column names to set - */ - public void setColNames(List colNames) { - this.colNames = colNames; - } } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/RuleRegExp.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/RuleRegExp.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/RuleRegExp.java (revision 0) @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.util.Stack; +import java.util.Iterator; +import java.util.regex.Pattern; +import java.util.regex.Matcher; +import java.io.Serializable; + +import org.apache.hadoop.hive.ql.exec.Operator; + +/** + * Rule interface for Operators + * Used in operator dispatching to dispatch process/visitor functions for operators + */ +public class RuleRegExp implements Rule { + + private String ruleName; + private String regExp; + private Pattern pattern; + + /** + * + * @param stack the operators encountered so far + * @return operators names in the stack in the form of a string + **/ + private String getOpNameString(Stack> stack) { + String name = new String(); + Iterator> iter = stack.iterator(); + while (iter.hasNext()) { + Operator op = iter.next(); + name = name.concat(op.getOperatorName()); + } + + return name; + } + + /** + * The rule specified by the regular expression. Note that, the regular expression is specified in terms of operator + * name. For eg: TS.*RS -> means TableScan operator followed by anything any number of times followed by ReduceSink + * @param ruleName name of the rule + * @param regExp regular expression for the rule + **/ + public RuleRegExp(String ruleName, String regExp) { + this.ruleName = ruleName; + this.regExp = regExp; + pattern = Pattern.compile(regExp); + } + + /** + * This function returns the cost of the rule for the specified stack. Lower the cost, the better the rule is matched + * @param stacl operator stack encountered so far + * @return cost of the function + * @throws SemanticException + */ + public int cost(Stack> stack) throws SemanticException { + String opStackName = getOpNameString(stack); + Matcher m = pattern.matcher(opStackName); + if (m.matches()) + return m.group().length(); + + return Integer.MAX_VALUE; + } + + /** + * @return the name of the operator + **/ + public String getName() { + return ruleName; + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultRuleDispatcher.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultRuleDispatcher.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultRuleDispatcher.java (revision 0) @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.io.Serializable; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.ClassNotFoundException; +import java.util.List; +import java.util.Map; +import java.util.Stack; +import java.lang.ClassNotFoundException; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext; +import org.apache.hadoop.hive.ql.exec.OperatorFactory; +import org.apache.hadoop.hive.ql.exec.OperatorFactory.opTuple; + +/** + * Dispatches calls to relevant method in processor. The user registers various rules with the dispatcher, and + * the processor corresponding to closest matching rule is fired. + */ +public class DefaultRuleDispatcher implements Dispatcher { + + private Map opProcRules; + private OperatorProcessorContext opProcCtx; + + /** + * constructor + * @param opp operator processor that handles actual processing of the node + * @param opProcCtx operator processor context, which is opaque to the dispatcher + */ + public DefaultRuleDispatcher(Map opp, OperatorProcessorContext opProcCtx) { + this.opProcRules = opp; + this.opProcCtx = opProcCtx; + } + + /** + * dispatcher function + * @param op operator to process + * @param opStack the operators encountered so far + * @throws SemanticException + */ + public void dispatch(Operator op, Stack> opStack) + throws SemanticException { + + // find the firing rule + // find the rule from the stack specified + Rule rule = null; + int minCost = Integer.MAX_VALUE; + for (Rule r : opProcRules.keySet()) { + int cost = r.cost(opStack); + if (cost <= minCost) { + minCost = cost; + rule = r; + } + } + + assert rule != null; + OperatorProcessor proc = opProcRules.get(rule); + + // If the processor has registered a process method for the particular operator, invoke it. + // Otherwise implement the generic function, which would definitely be implemented + for(opTuple opt : OperatorFactory.opvec) { + if(opt.opClass.isInstance(op)) { + Method pcall; + try { + pcall = proc.getClass().getMethod("process", opt.opClass, + Class.forName("org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext")); + pcall.invoke(proc, op, opProcCtx); + return; + } catch (SecurityException e) { + assert false; + } catch (NoSuchMethodException e) { + assert false; + } catch (IllegalArgumentException e) { + assert false; + } catch (IllegalAccessException e) { + assert false; + } catch (InvocationTargetException e) { + throw new SemanticException(e.getTargetException()); + } catch (ClassNotFoundException e) { + assert false; + } + } + } + + try { + // no method found - invoke the generic function + Method pcall = proc.getClass().getMethod("process", Class.forName("org.apache.hadoop.hive.ql.exec.Operator"), + Class.forName("org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext")); + pcall.invoke(proc, ((Operator)op), opProcCtx); + + } catch (SecurityException e) { + assert false; + } catch (NoSuchMethodException e) { + assert false; + } catch (IllegalArgumentException e) { + assert false; + } catch (IllegalAccessException e) { + assert false; + } catch (InvocationTargetException e) { + throw new SemanticException(e.getTargetException()); + } catch (ClassNotFoundException e) { + assert false; + } + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/Dispatcher.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/Dispatcher.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/Dispatcher.java (revision 0) @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.io.Serializable; +import java.util.Stack; + +import org.apache.hadoop.hive.ql.exec.Operator; + +/** + * Dispatcher interface for Operators + * Used in operator graph walking to dispatch process/visitor functions for operators + */ +public interface Dispatcher { + + /** + * dispatcher function + * @param op operator to process + * @param Stack operator stack to process + * @throws SemanticException + */ + public abstract void dispatch(Operator op, Stack> stack) + throws SemanticException; +} Property changes on: ql/src/java/org/apache/hadoop/hive/ql/parse/Dispatcher.java ___________________________________________________________________ Added: svn:executable + * Index: ql/src/java/org/apache/hadoop/hive/ql/parse/OperatorProcessor.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/OperatorProcessor.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/OperatorProcessor.java (revision 0) @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse; + +import java.io.Serializable; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext; + +/** + * Base class for processing operators which is no-op. The specific processors can register their own context with + * the dispatcher. + */ +public interface OperatorProcessor { + + /** + * generic process for all ops that don't have specific implementations + * @param op operator to process + * @param opProcCtx operator processor context + * @throws SemanticException + */ + public void process(Operator op, OperatorProcessorContext opProcCtx) + throws SemanticException; +} Property changes on: ql/src/java/org/apache/hadoop/hive/ql/parse/OperatorProcessor.java ___________________________________________________________________ Added: svn:executable + * Index: ql/src/java/org/apache/hadoop/hive/ql/plan/groupByDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/groupByDesc.java (revision 722634) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/groupByDesc.java (working copy) @@ -70,7 +70,7 @@ this.keys = keys; } - @explain(displayName="") + @explain(displayName="aggregations") public java.util.ArrayList getAggregators() { return this.aggregators; } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/tableScanDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/tableScanDesc.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/tableScanDesc.java (revision 0) @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan; + +import java.io.Serializable; + +/** + * Table Scan Descriptor + * Currently, data is only read from a base source as part of map-reduce framework. So, nothing is stored in the + * descriptor. But, more things will be added here as table scan is invoked as part of local work. + **/ +@explain(displayName="TableScan") +public class tableScanDesc implements Serializable { + private static final long serialVersionUID = 1L; + @SuppressWarnings("nls") + public tableScanDesc() { + throw new RuntimeException("This class does not need to be instantiated"); + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (revision 722634) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (working copy) @@ -102,7 +102,7 @@ /** * Generate the table descriptor of DynamicSerDe and TBinarySortableProtocol. */ - public static tableDesc getBinarySortableTableDesc(List fieldSchemas) { + public static tableDesc getBinarySortableTableDesc(List fieldSchemas, String order) { String structName = "binary_sortable_table"; return new tableDesc( DynamicSerDe.class, @@ -110,10 +110,13 @@ SequenceFileOutputFormat.class, Utilities.makeProperties( "name", structName, - org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, TBinarySortableProtocol.class.getName(), + org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, + TBinarySortableProtocol.class.getName(), org.apache.hadoop.hive.serde.Constants.SERIALIZATION_DDL, - MetaStoreUtils.getDDLFromFieldSchema(structName, fieldSchemas) - )); + MetaStoreUtils.getDDLFromFieldSchema(structName, fieldSchemas), + org.apache.hadoop.hive.serde.Constants.SERIALIZATION_SORT_ORDER, + order + )); } /** @@ -182,11 +185,12 @@ public static reduceSinkDesc getReduceSinkDesc(ArrayList keyCols, ArrayList valueCols, int tag, - ArrayList partitionCols, + ArrayList partitionCols, + String order, int numReducers, boolean inferNumReducers) { return new reduceSinkDesc(keyCols, valueCols, tag, partitionCols, numReducers, inferNumReducers, - getBinarySortableTableDesc(getFieldSchemasFromColumnList(keyCols, "reducesinkkey")), + getBinarySortableTableDesc(getFieldSchemasFromColumnList(keyCols, "reducesinkkey"), order), getBinaryTableDesc(getFieldSchemasFromColumnList(valueCols, "reducesinkvalue"))); } @@ -221,7 +225,12 @@ partitionCols.add(SemanticAnalyzer.getFuncExprNodeDesc("rand")); } - return getReduceSinkDesc(keyCols, valueCols, tag, partitionCols, numReducers, inferNumReducers); + StringBuilder order = new StringBuilder(); + for (int i=0; i