Index: conf/hive-default.xml =================================================================== --- conf/hive-default.xml (revision 747596) +++ conf/hive-default.xml (working copy) @@ -72,7 +72,7 @@ hive.map.aggr - false + true Whether to use map-side aggregation in Hive Group By queries Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 747596) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -105,7 +105,8 @@ HIVEPARTITIONNAME("hive.partition.name", ""), HIVEPARTITIONPRUNER("hive.partition.pruning", "nonstrict"), HIVEALIAS("hive.alias", ""), - HIVEMAPSIDEAGGREGATE("hive.map.aggr", "false"), + HIVEMAPSIDEAGGREGATE("hive.map.aggr", "true"), + HIVEGROUPBYSKEW("hive.groupby.skewindata", "false"), HIVEJOINEMITINTERVAL("hive.join.emit.interval", 1000), HIVEMAPAGGRHASHMEMORY("hive.map.aggr.hash.percentmemory", (float)0.5), Index: ql/src/test/results/clientpositive/notable_alias2.q.out =================================================================== --- ql/src/test/results/clientpositive/notable_alias2.q.out (revision 747596) +++ ql/src/test/results/clientpositive/notable_alias2.q.out (working copy) @@ -3,8 +3,7 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 @@ -19,18 +18,25 @@ predicate: expr: (UDFToDouble(0) < UDFToDouble(100)) type: boolean - Reduce Output Operator - key expressions: + Group By Operator + aggregations: + expr: count(1) + keys: expr: 0 type: string - sort order: + - Map-reduce partition columns: - expr: rand() - type: double - tag: -1 - value expressions: - expr: 1 - type: int + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: bigint Reduce Operator Tree: Group By Operator aggregations: @@ -38,37 +44,6 @@ keys: expr: KEY.0 type: string - mode: partial1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.mapred.SequenceFileOutputFormat - name: binary_table - - Stage: Stage-2 - Map Reduce - Alias -> Map Operator Tree: - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/95316349/1574244180.10001 - Reduce Output Operator - key expressions: - expr: 0 - type: string - sort order: + - Map-reduce partition columns: - expr: 0 - type: string - tag: -1 - value expressions: - expr: 1 - type: bigint - Reduce Operator Tree: - Group By Operator - aggregations: - expr: count(VALUE.0) - keys: - expr: KEY.0 - type: string mode: final Select Operator expressions: Index: ql/src/test/results/clientpositive/groupby2_noskew.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby2_noskew.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby2_noskew.q.out (revision 0) @@ -0,0 +1,69 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest_g2)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION substr (TOK_COLREF src key) 1 1)) (TOK_SELEXPR (TOK_FUNCTIONDI count (TOK_FUNCTION substr (TOK_COLREF src value) 5))) (TOK_SELEXPR (TOK_FUNCTION concat (TOK_FUNCTION substr (TOK_COLREF src key) 1 1) (TOK_FUNCTION sum (TOK_FUNCTION substr (TOK_COLREF src value) 5))))) (TOK_GROUPBY (TOK_FUNCTION substr (TOK_COLREF src key) 1 1)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Reduce Output Operator + key expressions: + expr: substr(key, 1, 1) + type: string + expr: substr(value, 5) + type: string + sort order: ++ + Map-reduce partition columns: + expr: substr(key, 1, 1) + type: string + tag: -1 + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(UDFToDouble(KEY.1)) + expr: count(DISTINCT KEY.1) + keys: + expr: KEY.0 + type: string + mode: complete + Select Operator + expressions: + expr: 0 + type: string + expr: 2 + type: bigint + expr: concat(0, UDFToString(1)) + type: string + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest_g2 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest_g2 + + +0 1 00.0 +1 71 116414.0 +2 69 225571.0 +3 62 332004.0 +4 74 452763.0 +5 6 5397.0 +6 5 6398.0 +7 6 7735.0 +8 8 8762.0 +9 7 91047.0 Index: ql/src/test/results/clientpositive/groupby5_map_skew.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby5_map_skew.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby5_map_skew.q.out (revision 0) @@ -0,0 +1,55 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_COLREF src key)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Select Operator + expressions: + expr: key + type: string + Group By Operator + aggregations: + expr: sum(UDFToDouble(0)) + mode: hash + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: 0 + type: double + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(VALUE.0) + mode: final + Select Operator + expressions: + expr: 0 + type: double + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + +NULL Index: ql/src/test/results/clientpositive/groupby4_map_skew.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby4_map_skew.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby4_map_skew.q.out (revision 0) @@ -0,0 +1,52 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION count 1))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Select Operator + Group By Operator + aggregations: + expr: count(1) + mode: hash + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: 0 + type: bigint + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(VALUE.0) + mode: final + Select Operator + expressions: + expr: 0 + type: bigint + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + +500 Index: ql/src/test/results/clientpositive/groupby7_noskew.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby7_noskew.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby7_noskew.q.out (revision 0) @@ -0,0 +1,727 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF SRC)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB DEST1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF SRC key)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION SUBSTR (TOK_COLREF SRC value) 5)))) (TOK_GROUPBY (TOK_COLREF SRC key))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB DEST2)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF SRC key)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION SUBSTR (TOK_COLREF SRC value) 5)))) (TOK_GROUPBY (TOK_COLREF SRC key)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1, Stage-2 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-1, Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: -1 + value expressions: + expr: substr(value, 5) + type: string + File Output Operator + compressed: true + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(UDFToDouble(VALUE.0)) + keys: + expr: KEY.0 + type: string + mode: complete + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: double + File Output Operator + compressed: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest2 + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + /data/users/njain/hive1/hive/build/ql/tmp/3382765/950544050.10002 + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: -1 + value expressions: + expr: substr(value, 5) + type: string + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(UDFToDouble(VALUE.0)) + keys: + expr: KEY.0 + type: string + mode: complete + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: double + File Output Operator + compressed: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest2 + + +0 0.0 +10 10.0 +100 200.0 +103 206.0 +104 208.0 +105 105.0 +11 11.0 +111 111.0 +113 226.0 +114 114.0 +116 116.0 +118 236.0 +119 357.0 +12 24.0 +120 240.0 +125 250.0 +126 126.0 +128 384.0 +129 258.0 +131 131.0 +133 133.0 +134 268.0 +136 136.0 +137 274.0 +138 552.0 +143 143.0 +145 145.0 +146 292.0 +149 298.0 +15 30.0 +150 150.0 +152 304.0 +153 153.0 +155 155.0 +156 156.0 +157 157.0 +158 158.0 +160 160.0 +162 162.0 +163 163.0 +164 328.0 +165 330.0 +166 166.0 +167 501.0 +168 168.0 +169 676.0 +17 17.0 +170 170.0 +172 344.0 +174 348.0 +175 350.0 +176 352.0 +177 177.0 +178 178.0 +179 358.0 +18 36.0 +180 180.0 +181 181.0 +183 183.0 +186 186.0 +187 561.0 +189 189.0 +19 19.0 +190 190.0 +191 382.0 +192 192.0 +193 579.0 +194 194.0 +195 390.0 +196 196.0 +197 394.0 +199 597.0 +2 2.0 +20 20.0 +200 400.0 +201 201.0 +202 202.0 +203 406.0 +205 410.0 +207 414.0 +208 624.0 +209 418.0 +213 426.0 +214 214.0 +216 432.0 +217 434.0 +218 218.0 +219 438.0 +221 442.0 +222 222.0 +223 446.0 +224 448.0 +226 226.0 +228 228.0 +229 458.0 +230 1150.0 +233 466.0 +235 235.0 +237 474.0 +238 476.0 +239 478.0 +24 48.0 +241 241.0 +242 484.0 +244 244.0 +247 247.0 +248 248.0 +249 249.0 +252 252.0 +255 510.0 +256 512.0 +257 257.0 +258 258.0 +26 52.0 +260 260.0 +262 262.0 +263 263.0 +265 530.0 +266 266.0 +27 27.0 +272 544.0 +273 819.0 +274 274.0 +275 275.0 +277 1108.0 +278 556.0 +28 28.0 +280 560.0 +281 562.0 +282 564.0 +283 283.0 +284 284.0 +285 285.0 +286 286.0 +287 287.0 +288 576.0 +289 289.0 +291 291.0 +292 292.0 +296 296.0 +298 894.0 +30 30.0 +302 302.0 +305 305.0 +306 306.0 +307 614.0 +308 308.0 +309 618.0 +310 310.0 +311 933.0 +315 315.0 +316 948.0 +317 634.0 +318 954.0 +321 642.0 +322 644.0 +323 323.0 +325 650.0 +327 981.0 +33 33.0 +331 662.0 +332 332.0 +333 666.0 +335 335.0 +336 336.0 +338 338.0 +339 339.0 +34 34.0 +341 341.0 +342 684.0 +344 688.0 +345 345.0 +348 1740.0 +35 105.0 +351 351.0 +353 706.0 +356 356.0 +360 360.0 +362 362.0 +364 364.0 +365 365.0 +366 366.0 +367 734.0 +368 368.0 +369 1107.0 +37 74.0 +373 373.0 +374 374.0 +375 375.0 +377 377.0 +378 378.0 +379 379.0 +382 764.0 +384 1152.0 +386 386.0 +389 389.0 +392 392.0 +393 393.0 +394 394.0 +395 790.0 +396 1188.0 +397 794.0 +399 798.0 +4 4.0 +400 400.0 +401 2005.0 +402 402.0 +403 1209.0 +404 808.0 +406 1624.0 +407 407.0 +409 1227.0 +41 41.0 +411 411.0 +413 826.0 +414 828.0 +417 1251.0 +418 418.0 +419 419.0 +42 84.0 +421 421.0 +424 848.0 +427 427.0 +429 858.0 +43 43.0 +430 1290.0 +431 1293.0 +432 432.0 +435 435.0 +436 436.0 +437 437.0 +438 1314.0 +439 878.0 +44 44.0 +443 443.0 +444 444.0 +446 446.0 +448 448.0 +449 449.0 +452 452.0 +453 453.0 +454 1362.0 +455 455.0 +457 457.0 +458 916.0 +459 918.0 +460 460.0 +462 924.0 +463 926.0 +466 1398.0 +467 467.0 +468 1872.0 +469 2345.0 +47 47.0 +470 470.0 +472 472.0 +475 475.0 +477 477.0 +478 956.0 +479 479.0 +480 1440.0 +481 481.0 +482 482.0 +483 483.0 +484 484.0 +485 485.0 +487 487.0 +489 1956.0 +490 490.0 +491 491.0 +492 984.0 +493 493.0 +494 494.0 +495 495.0 +496 496.0 +497 497.0 +498 1494.0 +5 15.0 +51 102.0 +53 53.0 +54 54.0 +57 57.0 +58 116.0 +64 64.0 +65 65.0 +66 66.0 +67 134.0 +69 69.0 +70 210.0 +72 144.0 +74 74.0 +76 152.0 +77 77.0 +78 78.0 +8 8.0 +80 80.0 +82 82.0 +83 166.0 +84 168.0 +85 85.0 +86 86.0 +87 87.0 +9 9.0 +90 270.0 +92 92.0 +95 190.0 +96 96.0 +97 194.0 +98 196.0 +0 0.0 +10 10.0 +100 200.0 +103 206.0 +104 208.0 +105 105.0 +11 11.0 +111 111.0 +113 226.0 +114 114.0 +116 116.0 +118 236.0 +119 357.0 +12 24.0 +120 240.0 +125 250.0 +126 126.0 +128 384.0 +129 258.0 +131 131.0 +133 133.0 +134 268.0 +136 136.0 +137 274.0 +138 552.0 +143 143.0 +145 145.0 +146 292.0 +149 298.0 +15 30.0 +150 150.0 +152 304.0 +153 153.0 +155 155.0 +156 156.0 +157 157.0 +158 158.0 +160 160.0 +162 162.0 +163 163.0 +164 328.0 +165 330.0 +166 166.0 +167 501.0 +168 168.0 +169 676.0 +17 17.0 +170 170.0 +172 344.0 +174 348.0 +175 350.0 +176 352.0 +177 177.0 +178 178.0 +179 358.0 +18 36.0 +180 180.0 +181 181.0 +183 183.0 +186 186.0 +187 561.0 +189 189.0 +19 19.0 +190 190.0 +191 382.0 +192 192.0 +193 579.0 +194 194.0 +195 390.0 +196 196.0 +197 394.0 +199 597.0 +2 2.0 +20 20.0 +200 400.0 +201 201.0 +202 202.0 +203 406.0 +205 410.0 +207 414.0 +208 624.0 +209 418.0 +213 426.0 +214 214.0 +216 432.0 +217 434.0 +218 218.0 +219 438.0 +221 442.0 +222 222.0 +223 446.0 +224 448.0 +226 226.0 +228 228.0 +229 458.0 +230 1150.0 +233 466.0 +235 235.0 +237 474.0 +238 476.0 +239 478.0 +24 48.0 +241 241.0 +242 484.0 +244 244.0 +247 247.0 +248 248.0 +249 249.0 +252 252.0 +255 510.0 +256 512.0 +257 257.0 +258 258.0 +26 52.0 +260 260.0 +262 262.0 +263 263.0 +265 530.0 +266 266.0 +27 27.0 +272 544.0 +273 819.0 +274 274.0 +275 275.0 +277 1108.0 +278 556.0 +28 28.0 +280 560.0 +281 562.0 +282 564.0 +283 283.0 +284 284.0 +285 285.0 +286 286.0 +287 287.0 +288 576.0 +289 289.0 +291 291.0 +292 292.0 +296 296.0 +298 894.0 +30 30.0 +302 302.0 +305 305.0 +306 306.0 +307 614.0 +308 308.0 +309 618.0 +310 310.0 +311 933.0 +315 315.0 +316 948.0 +317 634.0 +318 954.0 +321 642.0 +322 644.0 +323 323.0 +325 650.0 +327 981.0 +33 33.0 +331 662.0 +332 332.0 +333 666.0 +335 335.0 +336 336.0 +338 338.0 +339 339.0 +34 34.0 +341 341.0 +342 684.0 +344 688.0 +345 345.0 +348 1740.0 +35 105.0 +351 351.0 +353 706.0 +356 356.0 +360 360.0 +362 362.0 +364 364.0 +365 365.0 +366 366.0 +367 734.0 +368 368.0 +369 1107.0 +37 74.0 +373 373.0 +374 374.0 +375 375.0 +377 377.0 +378 378.0 +379 379.0 +382 764.0 +384 1152.0 +386 386.0 +389 389.0 +392 392.0 +393 393.0 +394 394.0 +395 790.0 +396 1188.0 +397 794.0 +399 798.0 +4 4.0 +400 400.0 +401 2005.0 +402 402.0 +403 1209.0 +404 808.0 +406 1624.0 +407 407.0 +409 1227.0 +41 41.0 +411 411.0 +413 826.0 +414 828.0 +417 1251.0 +418 418.0 +419 419.0 +42 84.0 +421 421.0 +424 848.0 +427 427.0 +429 858.0 +43 43.0 +430 1290.0 +431 1293.0 +432 432.0 +435 435.0 +436 436.0 +437 437.0 +438 1314.0 +439 878.0 +44 44.0 +443 443.0 +444 444.0 +446 446.0 +448 448.0 +449 449.0 +452 452.0 +453 453.0 +454 1362.0 +455 455.0 +457 457.0 +458 916.0 +459 918.0 +460 460.0 +462 924.0 +463 926.0 +466 1398.0 +467 467.0 +468 1872.0 +469 2345.0 +47 47.0 +470 470.0 +472 472.0 +475 475.0 +477 477.0 +478 956.0 +479 479.0 +480 1440.0 +481 481.0 +482 482.0 +483 483.0 +484 484.0 +485 485.0 +487 487.0 +489 1956.0 +490 490.0 +491 491.0 +492 984.0 +493 493.0 +494 494.0 +495 495.0 +496 496.0 +497 497.0 +498 1494.0 +5 15.0 +51 102.0 +53 53.0 +54 54.0 +57 57.0 +58 116.0 +64 64.0 +65 65.0 +66 66.0 +67 134.0 +69 69.0 +70 210.0 +72 144.0 +74 74.0 +76 152.0 +77 77.0 +78 78.0 +8 8.0 +80 80.0 +82 82.0 +83 166.0 +84 168.0 +85 85.0 +86 86.0 +87 87.0 +9 9.0 +90 270.0 +92 92.0 +95 190.0 +96 96.0 +97 194.0 +98 196.0 Index: ql/src/test/results/clientpositive/groupby2_map.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby2_map.q.out (revision 747596) +++ ql/src/test/results/clientpositive/groupby2_map.q.out (working copy) @@ -3,8 +3,7 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 @@ -31,8 +30,6 @@ Map-reduce partition columns: expr: 0 type: string - expr: 1 - type: string tag: -1 value expressions: expr: 2 @@ -43,40 +40,6 @@ Group By Operator aggregations: expr: sum(VALUE.0) - expr: count(DISTINCT KEY.1) - keys: - expr: KEY.0 - type: string - mode: partial2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.mapred.SequenceFileOutputFormat - name: binary_table - - Stage: Stage-2 - Map Reduce - Alias -> Map Operator Tree: - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/303279525/67065402.10001 - Reduce Output Operator - key expressions: - expr: 0 - type: string - sort order: + - Map-reduce partition columns: - expr: 0 - type: string - tag: -1 - value expressions: - expr: 1 - type: double - expr: 2 - type: bigint - Reduce Operator Tree: - Group By Operator - aggregations: - expr: sum(VALUE.0) expr: count(VALUE.1) keys: expr: KEY.0 Index: ql/src/test/results/clientpositive/groupby6_map.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby6_map.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby6_map.q.out (revision 0) @@ -0,0 +1,69 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECTDI (TOK_SELEXPR (TOK_FUNCTION substr (TOK_COLREF src value) 5 1))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Select Operator + expressions: + expr: value + type: string + Group By Operator + keys: + expr: substr(0, 5, 1) + type: string + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + Reduce Operator Tree: + Group By Operator + keys: + expr: KEY.0 + type: string + mode: final + Select Operator + expressions: + expr: 0 + type: string + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 Index: ql/src/test/results/clientpositive/notable_alias1.q.out =================================================================== --- ql/src/test/results/clientpositive/notable_alias1.q.out (revision 747596) +++ ql/src/test/results/clientpositive/notable_alias1.q.out (working copy) @@ -3,8 +3,7 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 @@ -19,18 +18,25 @@ predicate: expr: (UDFToDouble(0) < UDFToDouble(100)) type: boolean - Reduce Output Operator - key expressions: + Group By Operator + aggregations: + expr: count(1) + keys: expr: 0 type: string - sort order: + - Map-reduce partition columns: - expr: rand() - type: double - tag: -1 - value expressions: - expr: 1 - type: int + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: bigint Reduce Operator Tree: Group By Operator aggregations: @@ -38,37 +44,6 @@ keys: expr: KEY.0 type: string - mode: partial1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.mapred.SequenceFileOutputFormat - name: binary_table - - Stage: Stage-2 - Map Reduce - Alias -> Map Operator Tree: - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/632452177/1849231580.10001 - Reduce Output Operator - key expressions: - expr: 0 - type: string - sort order: + - Map-reduce partition columns: - expr: 0 - type: string - tag: -1 - value expressions: - expr: 1 - type: bigint - Reduce Operator Tree: - Group By Operator - aggregations: - expr: count(VALUE.0) - keys: - expr: KEY.0 - type: string mode: final Select Operator expressions: Index: ql/src/test/results/clientpositive/binarysortable_1.q.out =================================================================== --- ql/src/test/results/clientpositive/binarysortable_1.q.out (revision 747596) +++ ql/src/test/results/clientpositive/binarysortable_1.q.out (working copy) @@ -3,7 +3,6 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 Stage-0 is a root stage STAGE PLANS: @@ -11,52 +10,28 @@ Map Reduce Alias -> Map Operator Tree: a:mytable - Reduce Output Operator - key expressions: + Group By Operator + aggregations: + expr: sum(UDFToDouble(value)) + keys: expr: key type: string - sort order: + - Map-reduce partition columns: - expr: rand() - type: double - tag: -1 - value expressions: - expr: value - type: string + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: double Reduce Operator Tree: Group By Operator aggregations: - expr: sum(UDFToDouble(VALUE.0)) - keys: - expr: KEY.0 - type: string - mode: partial1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.mapred.SequenceFileOutputFormat - name: binary_table - - Stage: Stage-2 - Map Reduce - Alias -> Map Operator Tree: - /data/users/zshao/sync/apache-trunk-HIVE-264/build/ql/tmp/33512536/104684401.10002 - Reduce Output Operator - key expressions: - expr: 0 - type: string - sort order: + - Map-reduce partition columns: - expr: 0 - type: string - tag: -1 - value expressions: - expr: 1 - type: double - Reduce Operator Tree: - Group By Operator - aggregations: expr: sum(VALUE.0) keys: expr: KEY.0 Index: ql/src/test/results/clientpositive/groupby3_map_skew.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby3_map_skew.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby3_map_skew.q.out (revision 0) @@ -0,0 +1,123 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION substr (TOK_COLREF src value) 5))) (TOK_SELEXPR (TOK_FUNCTION avg (TOK_FUNCTION substr (TOK_COLREF src value) 5))) (TOK_SELEXPR (TOK_FUNCTIONDI avg (TOK_FUNCTION substr (TOK_COLREF src value) 5))) (TOK_SELEXPR (TOK_FUNCTION max (TOK_FUNCTION substr (TOK_COLREF src value) 5))) (TOK_SELEXPR (TOK_FUNCTION min (TOK_FUNCTION substr (TOK_COLREF src value) 5)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Select Operator + expressions: + expr: value + type: string + Group By Operator + aggregations: + expr: sum(UDFToDouble(substr(0, 5))) + expr: max(substr(0, 5)) + expr: avg(DISTINCT UDFToDouble(substr(0, 5))) + expr: min(substr(0, 5)) + expr: avg(UDFToDouble(substr(0, 5))) + keys: + expr: substr(0, 5) + type: string + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: double + expr: 2 + type: string + expr: 3 + type: string + expr: 4 + type: string + expr: 5 + type: string + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(VALUE.0) + expr: max(VALUE.1) + expr: avg(DISTINCT UDFToDouble(KEY.0)) + expr: min(VALUE.3) + expr: avg(VALUE.4) + mode: partial2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + /data/users/njain/hive1/hive/build/ql/tmp/1268194131/1453611707.10001 + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: 0 + type: double + expr: 1 + type: string + expr: 2 + type: string + expr: 3 + type: string + expr: 4 + type: string + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(VALUE.0) + expr: max(VALUE.1) + expr: avg(VALUE.2) + expr: min(VALUE.3) + expr: avg(VALUE.4) + mode: final + Select Operator + expressions: + expr: 0 + type: double + expr: 4 + type: double + expr: 2 + type: double + expr: 1 + type: string + expr: 3 + type: string + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + +130091.0 260.182 256.10355987055016 98.0 0.0 Index: ql/src/test/results/clientpositive/groupby4_noskew.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby4_noskew.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby4_noskew.q.out (revision 0) @@ -0,0 +1,64 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION substr (TOK_COLREF src key) 1 1))) (TOK_GROUPBY (TOK_FUNCTION substr (TOK_COLREF src key) 1 1)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Select Operator + expressions: + expr: key + type: string + Reduce Output Operator + key expressions: + expr: substr(0, 1, 1) + type: string + sort order: + + Map-reduce partition columns: + expr: substr(0, 1, 1) + type: string + tag: -1 + Reduce Operator Tree: + Group By Operator + keys: + expr: KEY.0 + type: string + mode: complete + Select Operator + expressions: + expr: 0 + type: string + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 Index: ql/src/test/results/clientpositive/udf8.q.out =================================================================== --- ql/src/test/results/clientpositive/udf8.q.out (revision 747596) +++ ql/src/test/results/clientpositive/udf8.q.out (working copy) @@ -3,7 +3,6 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 Stage-0 is a root stage STAGE PLANS: @@ -11,46 +10,25 @@ Map Reduce Alias -> Map Operator Tree: dest1 - Reduce Output Operator - sort order: - Map-reduce partition columns: - expr: rand() - type: double - tag: -1 - value expressions: - expr: c1 - type: string + Group By Operator + aggregations: + expr: sum(UDFToDouble(c1)) + expr: count(c1) + expr: avg(UDFToDouble(c1)) + mode: hash + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: 0 + type: double + expr: 1 + type: bigint + expr: 2 + type: string Reduce Operator Tree: Group By Operator aggregations: - expr: sum(UDFToDouble(VALUE.0)) - expr: count(VALUE.0) - expr: avg(UDFToDouble(VALUE.0)) - mode: partial1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.mapred.SequenceFileOutputFormat - name: binary_table - - Stage: Stage-2 - Map Reduce - Alias -> Map Operator Tree: - /data/users/athusoo/commits/hive_trunk_ws2/build/ql/tmp/181599227/280831237.10002 - Reduce Output Operator - sort order: - tag: -1 - value expressions: - expr: 0 - type: double - expr: 1 - type: bigint - expr: 2 - type: string - Reduce Operator Tree: - Group By Operator - aggregations: expr: sum(VALUE.0) expr: count(VALUE.1) expr: avg(VALUE.2) Index: ql/src/test/results/clientpositive/groupby8.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby8.q.out (revision 747596) +++ ql/src/test/results/clientpositive/groupby8.q.out (working copy) @@ -1,3 +1,178 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF SRC)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB DEST1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF SRC key)) (TOK_SELEXPR (TOK_FUNCTIONDI COUNT (TOK_FUNCTION SUBSTR (TOK_COLREF SRC value) 5)))) (TOK_GROUPBY (TOK_COLREF SRC key))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB DEST2)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF SRC key)) (TOK_SELEXPR (TOK_FUNCTIONDI COUNT (TOK_FUNCTION SUBSTR (TOK_COLREF SRC value) 5)))) (TOK_GROUPBY (TOK_COLREF SRC key)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2, Stage-4 + Stage-3 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-2, Stage-4 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Reduce Output Operator + key expressions: + expr: key + type: string + expr: substr(value, 5) + type: string + sort order: ++ + Map-reduce partition columns: + expr: key + type: string + expr: substr(value, 5) + type: string + tag: -1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(DISTINCT KEY.1) + keys: + expr: KEY.0 + type: string + mode: partial1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + /data/users/njain/hive1/hive/build/ql/tmp/944730679/831792853.10002 + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: bigint + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: final + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: bigint + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest2 + + Stage: Stage-3 + Map Reduce + Alias -> Map Operator Tree: + /data/users/njain/hive1/hive/build/ql/tmp/944730679/831792853.10003 + Reduce Output Operator + key expressions: + expr: key + type: string + expr: substr(value, 5) + type: string + sort order: ++ + Map-reduce partition columns: + expr: key + type: string + expr: substr(value, 5) + type: string + tag: -1 + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(DISTINCT KEY.1) + keys: + expr: KEY.0 + type: string + mode: partial1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + + Stage: Stage-4 + Map Reduce + Alias -> Map Operator Tree: + /data/users/njain/hive1/hive/build/ql/tmp/944730679/831792853.10004 + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: bigint + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: final + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: bigint + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest2 + + 0 1 10 1 100 1 Index: ql/src/test/results/clientpositive/groupby1_noskew.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby1_noskew.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby1_noskew.q.out (revision 0) @@ -0,0 +1,366 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest_g1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src key)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION substr (TOK_COLREF src value) 5)))) (TOK_GROUPBY (TOK_COLREF src key)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: -1 + value expressions: + expr: substr(value, 5) + type: string + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(UDFToDouble(VALUE.0)) + keys: + expr: KEY.0 + type: string + mode: complete + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: double + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest_g1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest_g1 + + +0 0.0 +10 10.0 +100 200.0 +103 206.0 +104 208.0 +105 105.0 +11 11.0 +111 111.0 +113 226.0 +114 114.0 +116 116.0 +118 236.0 +119 357.0 +12 24.0 +120 240.0 +125 250.0 +126 126.0 +128 384.0 +129 258.0 +131 131.0 +133 133.0 +134 268.0 +136 136.0 +137 274.0 +138 552.0 +143 143.0 +145 145.0 +146 292.0 +149 298.0 +15 30.0 +150 150.0 +152 304.0 +153 153.0 +155 155.0 +156 156.0 +157 157.0 +158 158.0 +160 160.0 +162 162.0 +163 163.0 +164 328.0 +165 330.0 +166 166.0 +167 501.0 +168 168.0 +169 676.0 +17 17.0 +170 170.0 +172 344.0 +174 348.0 +175 350.0 +176 352.0 +177 177.0 +178 178.0 +179 358.0 +18 36.0 +180 180.0 +181 181.0 +183 183.0 +186 186.0 +187 561.0 +189 189.0 +19 19.0 +190 190.0 +191 382.0 +192 192.0 +193 579.0 +194 194.0 +195 390.0 +196 196.0 +197 394.0 +199 597.0 +2 2.0 +20 20.0 +200 400.0 +201 201.0 +202 202.0 +203 406.0 +205 410.0 +207 414.0 +208 624.0 +209 418.0 +213 426.0 +214 214.0 +216 432.0 +217 434.0 +218 218.0 +219 438.0 +221 442.0 +222 222.0 +223 446.0 +224 448.0 +226 226.0 +228 228.0 +229 458.0 +230 1150.0 +233 466.0 +235 235.0 +237 474.0 +238 476.0 +239 478.0 +24 48.0 +241 241.0 +242 484.0 +244 244.0 +247 247.0 +248 248.0 +249 249.0 +252 252.0 +255 510.0 +256 512.0 +257 257.0 +258 258.0 +26 52.0 +260 260.0 +262 262.0 +263 263.0 +265 530.0 +266 266.0 +27 27.0 +272 544.0 +273 819.0 +274 274.0 +275 275.0 +277 1108.0 +278 556.0 +28 28.0 +280 560.0 +281 562.0 +282 564.0 +283 283.0 +284 284.0 +285 285.0 +286 286.0 +287 287.0 +288 576.0 +289 289.0 +291 291.0 +292 292.0 +296 296.0 +298 894.0 +30 30.0 +302 302.0 +305 305.0 +306 306.0 +307 614.0 +308 308.0 +309 618.0 +310 310.0 +311 933.0 +315 315.0 +316 948.0 +317 634.0 +318 954.0 +321 642.0 +322 644.0 +323 323.0 +325 650.0 +327 981.0 +33 33.0 +331 662.0 +332 332.0 +333 666.0 +335 335.0 +336 336.0 +338 338.0 +339 339.0 +34 34.0 +341 341.0 +342 684.0 +344 688.0 +345 345.0 +348 1740.0 +35 105.0 +351 351.0 +353 706.0 +356 356.0 +360 360.0 +362 362.0 +364 364.0 +365 365.0 +366 366.0 +367 734.0 +368 368.0 +369 1107.0 +37 74.0 +373 373.0 +374 374.0 +375 375.0 +377 377.0 +378 378.0 +379 379.0 +382 764.0 +384 1152.0 +386 386.0 +389 389.0 +392 392.0 +393 393.0 +394 394.0 +395 790.0 +396 1188.0 +397 794.0 +399 798.0 +4 4.0 +400 400.0 +401 2005.0 +402 402.0 +403 1209.0 +404 808.0 +406 1624.0 +407 407.0 +409 1227.0 +41 41.0 +411 411.0 +413 826.0 +414 828.0 +417 1251.0 +418 418.0 +419 419.0 +42 84.0 +421 421.0 +424 848.0 +427 427.0 +429 858.0 +43 43.0 +430 1290.0 +431 1293.0 +432 432.0 +435 435.0 +436 436.0 +437 437.0 +438 1314.0 +439 878.0 +44 44.0 +443 443.0 +444 444.0 +446 446.0 +448 448.0 +449 449.0 +452 452.0 +453 453.0 +454 1362.0 +455 455.0 +457 457.0 +458 916.0 +459 918.0 +460 460.0 +462 924.0 +463 926.0 +466 1398.0 +467 467.0 +468 1872.0 +469 2345.0 +47 47.0 +470 470.0 +472 472.0 +475 475.0 +477 477.0 +478 956.0 +479 479.0 +480 1440.0 +481 481.0 +482 482.0 +483 483.0 +484 484.0 +485 485.0 +487 487.0 +489 1956.0 +490 490.0 +491 491.0 +492 984.0 +493 493.0 +494 494.0 +495 495.0 +496 496.0 +497 497.0 +498 1494.0 +5 15.0 +51 102.0 +53 53.0 +54 54.0 +57 57.0 +58 116.0 +64 64.0 +65 65.0 +66 66.0 +67 134.0 +69 69.0 +70 210.0 +72 144.0 +74 74.0 +76 152.0 +77 77.0 +78 78.0 +8 8.0 +80 80.0 +82 82.0 +83 166.0 +84 168.0 +85 85.0 +86 86.0 +87 87.0 +9 9.0 +90 270.0 +92 92.0 +95 190.0 +96 96.0 +97 194.0 +98 196.0 Index: ql/src/test/results/clientpositive/groupby2_map_skew.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby2_map_skew.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby2_map_skew.q.out (revision 0) @@ -0,0 +1,121 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION substr (TOK_COLREF src key) 1 1)) (TOK_SELEXPR (TOK_FUNCTIONDI count (TOK_FUNCTION substr (TOK_COLREF src value) 5))) (TOK_SELEXPR (TOK_FUNCTION concat (TOK_FUNCTION substr (TOK_COLREF src key) 1 1) (TOK_FUNCTION sum (TOK_FUNCTION substr (TOK_COLREF src value) 5))))) (TOK_GROUPBY (TOK_FUNCTION substr (TOK_COLREF src key) 1 1)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Group By Operator + aggregations: + expr: sum(UDFToDouble(substr(value, 5))) + expr: count(DISTINCT substr(value, 5)) + keys: + expr: substr(key, 1, 1) + type: string + expr: substr(value, 5) + type: string + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + expr: 1 + type: string + sort order: ++ + Map-reduce partition columns: + expr: 0 + type: string + expr: 1 + type: string + tag: -1 + value expressions: + expr: 2 + type: double + expr: 3 + type: bigint + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(VALUE.0) + expr: count(DISTINCT KEY.1) + keys: + expr: KEY.0 + type: string + mode: partial2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + /data/users/njain/hive1/hive/build/ql/tmp/65879383/505274375.10001 + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: double + expr: 2 + type: bigint + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(VALUE.0) + expr: count(VALUE.1) + keys: + expr: KEY.0 + type: string + mode: final + Select Operator + expressions: + expr: 0 + type: string + expr: 2 + type: bigint + expr: concat(0, UDFToString(1)) + type: string + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + +0 1 00.0 +1 71 116414.0 +2 69 225571.0 +3 62 332004.0 +4 74 452763.0 +5 6 5397.0 +6 5 6398.0 +7 6 7735.0 +8 8 8762.0 +9 7 91047.0 Index: ql/src/test/results/clientpositive/groupby2_limit.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby2_limit.q.out (revision 747596) +++ ql/src/test/results/clientpositive/groupby2_limit.q.out (working copy) @@ -3,7 +3,6 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 Stage-0 is a root stage STAGE PLANS: @@ -11,52 +10,28 @@ Map Reduce Alias -> Map Operator Tree: src - Reduce Output Operator - key expressions: + Group By Operator + aggregations: + expr: sum(UDFToDouble(substr(value, 5))) + keys: expr: key type: string - sort order: + - Map-reduce partition columns: - expr: rand() - type: double - tag: -1 - value expressions: - expr: substr(value, 5) - type: string + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: double Reduce Operator Tree: Group By Operator aggregations: - expr: sum(UDFToDouble(VALUE.0)) - keys: - expr: KEY.0 - type: string - mode: partial1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.mapred.SequenceFileOutputFormat - name: binary_table - - Stage: Stage-2 - Map Reduce - Alias -> Map Operator Tree: - /tmp/hive-zshao/42329253/238260652.10002 - Reduce Output Operator - key expressions: - expr: 0 - type: string - sort order: + - Map-reduce partition columns: - expr: 0 - type: string - tag: -1 - value expressions: - expr: 1 - type: double - Reduce Operator Tree: - Group By Operator - aggregations: expr: sum(VALUE.0) keys: expr: KEY.0 Index: ql/src/test/results/clientpositive/groupby3_map.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby3_map.q.out (revision 747596) +++ ql/src/test/results/clientpositive/groupby3_map.q.out (working copy) @@ -3,8 +3,7 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 @@ -31,9 +30,6 @@ expr: 0 type: string sort order: + - Map-reduce partition columns: - expr: 0 - type: string tag: -1 value expressions: expr: 1 @@ -51,40 +47,6 @@ aggregations: expr: sum(VALUE.0) expr: max(VALUE.1) - expr: avg(DISTINCT UDFToDouble(KEY.0)) - expr: min(VALUE.3) - expr: avg(VALUE.4) - mode: partial2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.mapred.SequenceFileOutputFormat - name: binary_table - - Stage: Stage-2 - Map Reduce - Alias -> Map Operator Tree: - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/766792078/1062753384.10001 - Reduce Output Operator - sort order: - tag: -1 - value expressions: - expr: 0 - type: double - expr: 1 - type: string - expr: 2 - type: string - expr: 3 - type: string - expr: 4 - type: string - Reduce Operator Tree: - Group By Operator - aggregations: - expr: sum(VALUE.0) - expr: max(VALUE.1) expr: avg(VALUE.2) expr: min(VALUE.3) expr: avg(VALUE.4) Index: ql/src/test/results/clientpositive/groupby7_map.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby7_map.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby7_map.q.out (revision 0) @@ -0,0 +1,741 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF SRC)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB DEST1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF SRC key)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION SUBSTR (TOK_COLREF SRC value) 5)))) (TOK_GROUPBY (TOK_COLREF SRC key))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB DEST2)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF SRC key)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION SUBSTR (TOK_COLREF SRC value) 5)))) (TOK_GROUPBY (TOK_COLREF SRC key)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1, Stage-2 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-1, Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Group By Operator + aggregations: + expr: sum(UDFToDouble(substr(value, 5))) + keys: + expr: key + type: string + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: double + Group By Operator + aggregations: + expr: sum(UDFToDouble(substr(value, 5))) + keys: + expr: key + type: string + mode: hash + File Output Operator + compressed: true + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: final + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: double + File Output Operator + compressed: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest2 + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + /data/users/njain/hive1/hive/build/ql/tmp/614671824/22403266.10002 + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: double + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: final + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: double + File Output Operator + compressed: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest2 + + +0 0.0 +10 10.0 +100 200.0 +103 206.0 +104 208.0 +105 105.0 +11 11.0 +111 111.0 +113 226.0 +114 114.0 +116 116.0 +118 236.0 +119 357.0 +12 24.0 +120 240.0 +125 250.0 +126 126.0 +128 384.0 +129 258.0 +131 131.0 +133 133.0 +134 268.0 +136 136.0 +137 274.0 +138 552.0 +143 143.0 +145 145.0 +146 292.0 +149 298.0 +15 30.0 +150 150.0 +152 304.0 +153 153.0 +155 155.0 +156 156.0 +157 157.0 +158 158.0 +160 160.0 +162 162.0 +163 163.0 +164 328.0 +165 330.0 +166 166.0 +167 501.0 +168 168.0 +169 676.0 +17 17.0 +170 170.0 +172 344.0 +174 348.0 +175 350.0 +176 352.0 +177 177.0 +178 178.0 +179 358.0 +18 36.0 +180 180.0 +181 181.0 +183 183.0 +186 186.0 +187 561.0 +189 189.0 +19 19.0 +190 190.0 +191 382.0 +192 192.0 +193 579.0 +194 194.0 +195 390.0 +196 196.0 +197 394.0 +199 597.0 +2 2.0 +20 20.0 +200 400.0 +201 201.0 +202 202.0 +203 406.0 +205 410.0 +207 414.0 +208 624.0 +209 418.0 +213 426.0 +214 214.0 +216 432.0 +217 434.0 +218 218.0 +219 438.0 +221 442.0 +222 222.0 +223 446.0 +224 448.0 +226 226.0 +228 228.0 +229 458.0 +230 1150.0 +233 466.0 +235 235.0 +237 474.0 +238 476.0 +239 478.0 +24 48.0 +241 241.0 +242 484.0 +244 244.0 +247 247.0 +248 248.0 +249 249.0 +252 252.0 +255 510.0 +256 512.0 +257 257.0 +258 258.0 +26 52.0 +260 260.0 +262 262.0 +263 263.0 +265 530.0 +266 266.0 +27 27.0 +272 544.0 +273 819.0 +274 274.0 +275 275.0 +277 1108.0 +278 556.0 +28 28.0 +280 560.0 +281 562.0 +282 564.0 +283 283.0 +284 284.0 +285 285.0 +286 286.0 +287 287.0 +288 576.0 +289 289.0 +291 291.0 +292 292.0 +296 296.0 +298 894.0 +30 30.0 +302 302.0 +305 305.0 +306 306.0 +307 614.0 +308 308.0 +309 618.0 +310 310.0 +311 933.0 +315 315.0 +316 948.0 +317 634.0 +318 954.0 +321 642.0 +322 644.0 +323 323.0 +325 650.0 +327 981.0 +33 33.0 +331 662.0 +332 332.0 +333 666.0 +335 335.0 +336 336.0 +338 338.0 +339 339.0 +34 34.0 +341 341.0 +342 684.0 +344 688.0 +345 345.0 +348 1740.0 +35 105.0 +351 351.0 +353 706.0 +356 356.0 +360 360.0 +362 362.0 +364 364.0 +365 365.0 +366 366.0 +367 734.0 +368 368.0 +369 1107.0 +37 74.0 +373 373.0 +374 374.0 +375 375.0 +377 377.0 +378 378.0 +379 379.0 +382 764.0 +384 1152.0 +386 386.0 +389 389.0 +392 392.0 +393 393.0 +394 394.0 +395 790.0 +396 1188.0 +397 794.0 +399 798.0 +4 4.0 +400 400.0 +401 2005.0 +402 402.0 +403 1209.0 +404 808.0 +406 1624.0 +407 407.0 +409 1227.0 +41 41.0 +411 411.0 +413 826.0 +414 828.0 +417 1251.0 +418 418.0 +419 419.0 +42 84.0 +421 421.0 +424 848.0 +427 427.0 +429 858.0 +43 43.0 +430 1290.0 +431 1293.0 +432 432.0 +435 435.0 +436 436.0 +437 437.0 +438 1314.0 +439 878.0 +44 44.0 +443 443.0 +444 444.0 +446 446.0 +448 448.0 +449 449.0 +452 452.0 +453 453.0 +454 1362.0 +455 455.0 +457 457.0 +458 916.0 +459 918.0 +460 460.0 +462 924.0 +463 926.0 +466 1398.0 +467 467.0 +468 1872.0 +469 2345.0 +47 47.0 +470 470.0 +472 472.0 +475 475.0 +477 477.0 +478 956.0 +479 479.0 +480 1440.0 +481 481.0 +482 482.0 +483 483.0 +484 484.0 +485 485.0 +487 487.0 +489 1956.0 +490 490.0 +491 491.0 +492 984.0 +493 493.0 +494 494.0 +495 495.0 +496 496.0 +497 497.0 +498 1494.0 +5 15.0 +51 102.0 +53 53.0 +54 54.0 +57 57.0 +58 116.0 +64 64.0 +65 65.0 +66 66.0 +67 134.0 +69 69.0 +70 210.0 +72 144.0 +74 74.0 +76 152.0 +77 77.0 +78 78.0 +8 8.0 +80 80.0 +82 82.0 +83 166.0 +84 168.0 +85 85.0 +86 86.0 +87 87.0 +9 9.0 +90 270.0 +92 92.0 +95 190.0 +96 96.0 +97 194.0 +98 196.0 +0 0.0 +10 10.0 +100 200.0 +103 206.0 +104 208.0 +105 105.0 +11 11.0 +111 111.0 +113 226.0 +114 114.0 +116 116.0 +118 236.0 +119 357.0 +12 24.0 +120 240.0 +125 250.0 +126 126.0 +128 384.0 +129 258.0 +131 131.0 +133 133.0 +134 268.0 +136 136.0 +137 274.0 +138 552.0 +143 143.0 +145 145.0 +146 292.0 +149 298.0 +15 30.0 +150 150.0 +152 304.0 +153 153.0 +155 155.0 +156 156.0 +157 157.0 +158 158.0 +160 160.0 +162 162.0 +163 163.0 +164 328.0 +165 330.0 +166 166.0 +167 501.0 +168 168.0 +169 676.0 +17 17.0 +170 170.0 +172 344.0 +174 348.0 +175 350.0 +176 352.0 +177 177.0 +178 178.0 +179 358.0 +18 36.0 +180 180.0 +181 181.0 +183 183.0 +186 186.0 +187 561.0 +189 189.0 +19 19.0 +190 190.0 +191 382.0 +192 192.0 +193 579.0 +194 194.0 +195 390.0 +196 196.0 +197 394.0 +199 597.0 +2 2.0 +20 20.0 +200 400.0 +201 201.0 +202 202.0 +203 406.0 +205 410.0 +207 414.0 +208 624.0 +209 418.0 +213 426.0 +214 214.0 +216 432.0 +217 434.0 +218 218.0 +219 438.0 +221 442.0 +222 222.0 +223 446.0 +224 448.0 +226 226.0 +228 228.0 +229 458.0 +230 1150.0 +233 466.0 +235 235.0 +237 474.0 +238 476.0 +239 478.0 +24 48.0 +241 241.0 +242 484.0 +244 244.0 +247 247.0 +248 248.0 +249 249.0 +252 252.0 +255 510.0 +256 512.0 +257 257.0 +258 258.0 +26 52.0 +260 260.0 +262 262.0 +263 263.0 +265 530.0 +266 266.0 +27 27.0 +272 544.0 +273 819.0 +274 274.0 +275 275.0 +277 1108.0 +278 556.0 +28 28.0 +280 560.0 +281 562.0 +282 564.0 +283 283.0 +284 284.0 +285 285.0 +286 286.0 +287 287.0 +288 576.0 +289 289.0 +291 291.0 +292 292.0 +296 296.0 +298 894.0 +30 30.0 +302 302.0 +305 305.0 +306 306.0 +307 614.0 +308 308.0 +309 618.0 +310 310.0 +311 933.0 +315 315.0 +316 948.0 +317 634.0 +318 954.0 +321 642.0 +322 644.0 +323 323.0 +325 650.0 +327 981.0 +33 33.0 +331 662.0 +332 332.0 +333 666.0 +335 335.0 +336 336.0 +338 338.0 +339 339.0 +34 34.0 +341 341.0 +342 684.0 +344 688.0 +345 345.0 +348 1740.0 +35 105.0 +351 351.0 +353 706.0 +356 356.0 +360 360.0 +362 362.0 +364 364.0 +365 365.0 +366 366.0 +367 734.0 +368 368.0 +369 1107.0 +37 74.0 +373 373.0 +374 374.0 +375 375.0 +377 377.0 +378 378.0 +379 379.0 +382 764.0 +384 1152.0 +386 386.0 +389 389.0 +392 392.0 +393 393.0 +394 394.0 +395 790.0 +396 1188.0 +397 794.0 +399 798.0 +4 4.0 +400 400.0 +401 2005.0 +402 402.0 +403 1209.0 +404 808.0 +406 1624.0 +407 407.0 +409 1227.0 +41 41.0 +411 411.0 +413 826.0 +414 828.0 +417 1251.0 +418 418.0 +419 419.0 +42 84.0 +421 421.0 +424 848.0 +427 427.0 +429 858.0 +43 43.0 +430 1290.0 +431 1293.0 +432 432.0 +435 435.0 +436 436.0 +437 437.0 +438 1314.0 +439 878.0 +44 44.0 +443 443.0 +444 444.0 +446 446.0 +448 448.0 +449 449.0 +452 452.0 +453 453.0 +454 1362.0 +455 455.0 +457 457.0 +458 916.0 +459 918.0 +460 460.0 +462 924.0 +463 926.0 +466 1398.0 +467 467.0 +468 1872.0 +469 2345.0 +47 47.0 +470 470.0 +472 472.0 +475 475.0 +477 477.0 +478 956.0 +479 479.0 +480 1440.0 +481 481.0 +482 482.0 +483 483.0 +484 484.0 +485 485.0 +487 487.0 +489 1956.0 +490 490.0 +491 491.0 +492 984.0 +493 493.0 +494 494.0 +495 495.0 +496 496.0 +497 497.0 +498 1494.0 +5 15.0 +51 102.0 +53 53.0 +54 54.0 +57 57.0 +58 116.0 +64 64.0 +65 65.0 +66 66.0 +67 134.0 +69 69.0 +70 210.0 +72 144.0 +74 74.0 +76 152.0 +77 77.0 +78 78.0 +8 8.0 +80 80.0 +82 82.0 +83 166.0 +84 168.0 +85 85.0 +86 86.0 +87 87.0 +9 9.0 +90 270.0 +92 92.0 +95 190.0 +96 96.0 +97 194.0 +98 196.0 Index: ql/src/test/results/clientpositive/groupby1_map_skew.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby1_map_skew.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby1_map_skew.q.out (revision 0) @@ -0,0 +1,405 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src key)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION substr (TOK_COLREF src value) 5)))) (TOK_GROUPBY (TOK_COLREF src key)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Group By Operator + aggregations: + expr: sum(UDFToDouble(substr(value, 5))) + keys: + expr: key + type: string + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: rand() + type: double + tag: -1 + value expressions: + expr: 1 + type: double + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: partial2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + /data/users/njain/hive1/hive/build/ql/tmp/160791799/408498972.10001 + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: double + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: final + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: double + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + +0 0.0 +10 10.0 +100 200.0 +103 206.0 +104 208.0 +105 105.0 +11 11.0 +111 111.0 +113 226.0 +114 114.0 +116 116.0 +118 236.0 +119 357.0 +12 24.0 +120 240.0 +125 250.0 +126 126.0 +128 384.0 +129 258.0 +131 131.0 +133 133.0 +134 268.0 +136 136.0 +137 274.0 +138 552.0 +143 143.0 +145 145.0 +146 292.0 +149 298.0 +15 30.0 +150 150.0 +152 304.0 +153 153.0 +155 155.0 +156 156.0 +157 157.0 +158 158.0 +160 160.0 +162 162.0 +163 163.0 +164 328.0 +165 330.0 +166 166.0 +167 501.0 +168 168.0 +169 676.0 +17 17.0 +170 170.0 +172 344.0 +174 348.0 +175 350.0 +176 352.0 +177 177.0 +178 178.0 +179 358.0 +18 36.0 +180 180.0 +181 181.0 +183 183.0 +186 186.0 +187 561.0 +189 189.0 +19 19.0 +190 190.0 +191 382.0 +192 192.0 +193 579.0 +194 194.0 +195 390.0 +196 196.0 +197 394.0 +199 597.0 +2 2.0 +20 20.0 +200 400.0 +201 201.0 +202 202.0 +203 406.0 +205 410.0 +207 414.0 +208 624.0 +209 418.0 +213 426.0 +214 214.0 +216 432.0 +217 434.0 +218 218.0 +219 438.0 +221 442.0 +222 222.0 +223 446.0 +224 448.0 +226 226.0 +228 228.0 +229 458.0 +230 1150.0 +233 466.0 +235 235.0 +237 474.0 +238 476.0 +239 478.0 +24 48.0 +241 241.0 +242 484.0 +244 244.0 +247 247.0 +248 248.0 +249 249.0 +252 252.0 +255 510.0 +256 512.0 +257 257.0 +258 258.0 +26 52.0 +260 260.0 +262 262.0 +263 263.0 +265 530.0 +266 266.0 +27 27.0 +272 544.0 +273 819.0 +274 274.0 +275 275.0 +277 1108.0 +278 556.0 +28 28.0 +280 560.0 +281 562.0 +282 564.0 +283 283.0 +284 284.0 +285 285.0 +286 286.0 +287 287.0 +288 576.0 +289 289.0 +291 291.0 +292 292.0 +296 296.0 +298 894.0 +30 30.0 +302 302.0 +305 305.0 +306 306.0 +307 614.0 +308 308.0 +309 618.0 +310 310.0 +311 933.0 +315 315.0 +316 948.0 +317 634.0 +318 954.0 +321 642.0 +322 644.0 +323 323.0 +325 650.0 +327 981.0 +33 33.0 +331 662.0 +332 332.0 +333 666.0 +335 335.0 +336 336.0 +338 338.0 +339 339.0 +34 34.0 +341 341.0 +342 684.0 +344 688.0 +345 345.0 +348 1740.0 +35 105.0 +351 351.0 +353 706.0 +356 356.0 +360 360.0 +362 362.0 +364 364.0 +365 365.0 +366 366.0 +367 734.0 +368 368.0 +369 1107.0 +37 74.0 +373 373.0 +374 374.0 +375 375.0 +377 377.0 +378 378.0 +379 379.0 +382 764.0 +384 1152.0 +386 386.0 +389 389.0 +392 392.0 +393 393.0 +394 394.0 +395 790.0 +396 1188.0 +397 794.0 +399 798.0 +4 4.0 +400 400.0 +401 2005.0 +402 402.0 +403 1209.0 +404 808.0 +406 1624.0 +407 407.0 +409 1227.0 +41 41.0 +411 411.0 +413 826.0 +414 828.0 +417 1251.0 +418 418.0 +419 419.0 +42 84.0 +421 421.0 +424 848.0 +427 427.0 +429 858.0 +43 43.0 +430 1290.0 +431 1293.0 +432 432.0 +435 435.0 +436 436.0 +437 437.0 +438 1314.0 +439 878.0 +44 44.0 +443 443.0 +444 444.0 +446 446.0 +448 448.0 +449 449.0 +452 452.0 +453 453.0 +454 1362.0 +455 455.0 +457 457.0 +458 916.0 +459 918.0 +460 460.0 +462 924.0 +463 926.0 +466 1398.0 +467 467.0 +468 1872.0 +469 2345.0 +47 47.0 +470 470.0 +472 472.0 +475 475.0 +477 477.0 +478 956.0 +479 479.0 +480 1440.0 +481 481.0 +482 482.0 +483 483.0 +484 484.0 +485 485.0 +487 487.0 +489 1956.0 +490 490.0 +491 491.0 +492 984.0 +493 493.0 +494 494.0 +495 495.0 +496 496.0 +497 497.0 +498 1494.0 +5 15.0 +51 102.0 +53 53.0 +54 54.0 +57 57.0 +58 116.0 +64 64.0 +65 65.0 +66 66.0 +67 134.0 +69 69.0 +70 210.0 +72 144.0 +74 74.0 +76 152.0 +77 77.0 +78 78.0 +8 8.0 +80 80.0 +82 82.0 +83 166.0 +84 168.0 +85 85.0 +86 86.0 +87 87.0 +9 9.0 +90 270.0 +92 92.0 +95 190.0 +96 96.0 +97 194.0 +98 196.0 Index: ql/src/test/results/clientpositive/udf3.q.out =================================================================== --- ql/src/test/results/clientpositive/udf3.q.out (revision 747596) +++ ql/src/test/results/clientpositive/udf3.q.out (working copy) @@ -3,8 +3,7 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 @@ -12,53 +11,32 @@ Alias -> Map Operator Tree: src Select Operator - Reduce Output Operator - sort order: - Map-reduce partition columns: - expr: rand() - type: double - tag: -1 - value expressions: - expr: UDFToInteger('') - type: int + Group By Operator + aggregations: + expr: max(UDFToInteger('')) + expr: avg(UDFToDouble(UDFToInteger(''))) + expr: count(UDFToInteger('')) + expr: sum(UDFToDouble(UDFToInteger(''))) + expr: min(UDFToInteger('')) + mode: hash + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: 0 + type: int + expr: 1 + type: string + expr: 2 + type: bigint + expr: 3 + type: double + expr: 4 + type: int Reduce Operator Tree: Group By Operator aggregations: expr: max(VALUE.0) - expr: avg(UDFToDouble(VALUE.0)) - expr: count(VALUE.0) - expr: sum(UDFToDouble(VALUE.0)) - expr: min(VALUE.0) - mode: partial1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.mapred.SequenceFileOutputFormat - name: binary_table - - Stage: Stage-2 - Map Reduce - Alias -> Map Operator Tree: - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/866504261/124742303.10001 - Reduce Output Operator - sort order: - tag: -1 - value expressions: - expr: 0 - type: int - expr: 1 - type: string - expr: 2 - type: bigint - expr: 3 - type: double - expr: 4 - type: int - Reduce Operator Tree: - Group By Operator - aggregations: - expr: max(VALUE.0) expr: avg(VALUE.1) expr: count(VALUE.2) expr: sum(VALUE.3) Index: ql/src/test/results/clientpositive/groupby6_noskew.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby6_noskew.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby6_noskew.q.out (revision 0) @@ -0,0 +1,64 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECTDI (TOK_SELEXPR (TOK_FUNCTION substr (TOK_COLREF src value) 5 1))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Select Operator + expressions: + expr: value + type: string + Reduce Output Operator + key expressions: + expr: substr(0, 5, 1) + type: string + sort order: + + Map-reduce partition columns: + expr: substr(0, 5, 1) + type: string + tag: -1 + Reduce Operator Tree: + Group By Operator + keys: + expr: KEY.0 + type: string + mode: complete + Select Operator + expressions: + expr: 0 + type: string + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 Index: ql/src/test/results/clientpositive/udf7.q.out =================================================================== --- ql/src/test/results/clientpositive/udf7.q.out (revision 747596) +++ ql/src/test/results/clientpositive/udf7.q.out (working copy) @@ -74,4 +74,4 @@ limit: -1 -1.0986122886681096 NULL NULL 1.0986122886681096 NULL NULL 1.584962500721156 NULL NULL 0.4771212547196623 NULL NULL 1.584962500721156 NULL NULL NULL -1.0 7.38905609893065 8.0 8.0 0.125 8.0 2.0 NaN 1.0 +1.0986122886681098 NULL NULL 1.0986122886681098 NULL NULL 1.5849625007211563 NULL NULL 0.47712125471966244 NULL NULL 1.5849625007211563 NULL NULL NULL -1.0 7.38905609893065 8.0 8.0 0.125 8.0 2.0 NaN 1.0 Index: ql/src/test/results/clientpositive/join18.q.out =================================================================== --- ql/src/test/results/clientpositive/join18.q.out (revision 747596) +++ ql/src/test/results/clientpositive/join18.q.out (working copy) @@ -3,11 +3,9 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-3 depends on stages: Stage-2, Stage-5 - Stage-4 is a root stage - Stage-5 depends on stages: Stage-4 - Stage-3 depends on stages: Stage-2, Stage-5 + Stage-2 depends on stages: Stage-1, Stage-3 + Stage-3 is a root stage + Stage-2 depends on stages: Stage-1, Stage-3 Stage-0 is a root stage STAGE PLANS: @@ -15,53 +13,32 @@ Map Reduce Alias -> Map Operator Tree: b:src2 - Reduce Output Operator - key expressions: + Group By Operator + aggregations: + expr: count(DISTINCT value) + keys: expr: key type: string expr: value type: string - sort order: ++ - Map-reduce partition columns: - expr: key - type: string - expr: value - type: string - tag: -1 + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + expr: 1 + type: string + sort order: ++ + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 2 + type: bigint Reduce Operator Tree: Group By Operator aggregations: - expr: count(DISTINCT KEY.1) - keys: - expr: KEY.0 - type: string - mode: partial1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.mapred.SequenceFileOutputFormat - name: binary_table - - Stage: Stage-2 - Map Reduce - Alias -> Map Operator Tree: - /tmp/hive-zshao/356667681/95990945.10002 - Reduce Output Operator - key expressions: - expr: 0 - type: string - sort order: + - Map-reduce partition columns: - expr: 0 - type: string - tag: -1 - value expressions: - expr: 1 - type: bigint - Reduce Operator Tree: - Group By Operator - aggregations: expr: count(VALUE.0) keys: expr: KEY.0 @@ -80,7 +57,7 @@ output format: org.apache.hadoop.mapred.SequenceFileOutputFormat name: binary_table - Stage: Stage-3 + Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: $INTNAME1 @@ -157,22 +134,29 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - Stage: Stage-4 + Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: a:src1 - Reduce Output Operator - key expressions: + Group By Operator + aggregations: + expr: count(value) + keys: expr: key type: string - sort order: + - Map-reduce partition columns: - expr: rand() - type: double - tag: -1 - value expressions: - expr: value - type: string + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: bigint Reduce Operator Tree: Group By Operator aggregations: @@ -180,37 +164,6 @@ keys: expr: KEY.0 type: string - mode: partial1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.mapred.SequenceFileOutputFormat - name: binary_table - - Stage: Stage-5 - Map Reduce - Alias -> Map Operator Tree: - /tmp/hive-zshao/356667681/95990945.10004 - Reduce Output Operator - key expressions: - expr: 0 - type: string - sort order: + - Map-reduce partition columns: - expr: 0 - type: string - tag: -1 - value expressions: - expr: 1 - type: bigint - Reduce Operator Tree: - Group By Operator - aggregations: - expr: count(VALUE.0) - keys: - expr: KEY.0 - type: string mode: final Select Operator expressions: Index: ql/src/test/results/clientpositive/union2.q.out =================================================================== --- ql/src/test/results/clientpositive/union2.q.out (revision 747596) +++ ql/src/test/results/clientpositive/union2.q.out (working copy) @@ -3,7 +3,6 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 Stage-0 is a root stage STAGE PLANS: @@ -17,15 +16,16 @@ type: string expr: value type: string - Reduce Output Operator - sort order: - Map-reduce partition columns: - expr: rand() - type: double - tag: -1 - value expressions: - expr: 1 - type: int + Group By Operator + aggregations: + expr: count(1) + mode: hash + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: 0 + type: bigint null-subquery2:unionsrc-subquery2:s2 Select Operator expressions: @@ -33,41 +33,20 @@ type: string expr: value type: string - Reduce Output Operator - sort order: - Map-reduce partition columns: - expr: rand() - type: double - tag: -1 - value expressions: - expr: 1 - type: int + Group By Operator + aggregations: + expr: count(1) + mode: hash + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: 0 + type: bigint Reduce Operator Tree: Group By Operator aggregations: expr: count(VALUE.0) - mode: partial1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.mapred.SequenceFileOutputFormat - name: binary_table - - Stage: Stage-2 - Map Reduce - Alias -> Map Operator Tree: - /tmp/hive-zshao/175321401/121204523.10002 - Reduce Output Operator - sort order: - tag: -1 - value expressions: - expr: 0 - type: bigint - Reduce Operator Tree: - Group By Operator - aggregations: - expr: count(VALUE.0) mode: final Select Operator expressions: Index: ql/src/test/results/clientpositive/subq2.q.out =================================================================== --- ql/src/test/results/clientpositive/subq2.q.out (revision 747596) +++ ql/src/test/results/clientpositive/subq2.q.out (working copy) @@ -3,7 +3,6 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 Stage-0 is a root stage STAGE PLANS: @@ -15,18 +14,25 @@ expressions: expr: key type: string - Reduce Output Operator - key expressions: + Group By Operator + aggregations: + expr: count(1) + keys: expr: 0 type: string - sort order: + - Map-reduce partition columns: - expr: rand() - type: double - tag: -1 - value expressions: - expr: 1 - type: int + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: bigint Reduce Operator Tree: Group By Operator aggregations: @@ -34,37 +40,6 @@ keys: expr: KEY.0 type: string - mode: partial1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.mapred.SequenceFileOutputFormat - name: binary_table - - Stage: Stage-2 - Map Reduce - Alias -> Map Operator Tree: - /data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/tmp/217677179/137363579.10002 - Reduce Output Operator - key expressions: - expr: 0 - type: string - sort order: + - Map-reduce partition columns: - expr: 0 - type: string - tag: -1 - value expressions: - expr: 1 - type: bigint - Reduce Operator Tree: - Group By Operator - aggregations: - expr: count(VALUE.0) - keys: - expr: KEY.0 - type: string mode: final Select Operator expressions: Index: ql/src/test/results/clientpositive/groupby3_noskew.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby3_noskew.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby3_noskew.q.out (revision 0) @@ -0,0 +1,63 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION substr (TOK_COLREF src value) 5))) (TOK_SELEXPR (TOK_FUNCTION avg (TOK_FUNCTION substr (TOK_COLREF src value) 5))) (TOK_SELEXPR (TOK_FUNCTIONDI avg (TOK_FUNCTION substr (TOK_COLREF src value) 5))) (TOK_SELEXPR (TOK_FUNCTION max (TOK_FUNCTION substr (TOK_COLREF src value) 5))) (TOK_SELEXPR (TOK_FUNCTION min (TOK_FUNCTION substr (TOK_COLREF src value) 5)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Select Operator + expressions: + expr: value + type: string + Reduce Output Operator + key expressions: + expr: substr(0, 5) + type: string + sort order: + + tag: -1 + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(UDFToDouble(KEY.0)) + expr: max(KEY.0) + expr: avg(DISTINCT UDFToDouble(KEY.0)) + expr: min(KEY.0) + expr: avg(UDFToDouble(KEY.0)) + mode: complete + Select Operator + expressions: + expr: 0 + type: double + expr: 4 + type: double + expr: 2 + type: double + expr: 1 + type: string + expr: 3 + type: string + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + +130091.0 260.182 256.10355987055016 98.0 0.0 Index: ql/src/test/results/clientpositive/groupby8_map_skew.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby8_map_skew.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby8_map_skew.q.out (revision 0) @@ -0,0 +1,817 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF SRC)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB DEST1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF SRC key)) (TOK_SELEXPR (TOK_FUNCTIONDI COUNT (TOK_FUNCTION SUBSTR (TOK_COLREF SRC value) 5)))) (TOK_GROUPBY (TOK_COLREF SRC key))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB DEST2)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF SRC key)) (TOK_SELEXPR (TOK_FUNCTIONDI COUNT (TOK_FUNCTION SUBSTR (TOK_COLREF SRC value) 5)))) (TOK_GROUPBY (TOK_COLREF SRC key)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2, Stage-4 + Stage-3 depends on stages: Stage-2 + Stage-4 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-2, Stage-4 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Group By Operator + aggregations: + expr: count(DISTINCT substr(value, 5)) + keys: + expr: key + type: string + expr: substr(value, 5) + type: string + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + expr: 1 + type: string + sort order: ++ + Map-reduce partition columns: + expr: 0 + type: string + expr: 1 + type: string + tag: -1 + value expressions: + expr: 2 + type: bigint + Group By Operator + aggregations: + expr: count(DISTINCT substr(value, 5)) + keys: + expr: key + type: string + expr: substr(value, 5) + type: string + mode: hash + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(DISTINCT KEY.1) + keys: + expr: KEY.0 + type: string + mode: partial2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + /data/users/njain/hive1/hive/build/ql/tmp/249341865/269250345.10002 + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: bigint + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: final + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: bigint + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest2 + + Stage: Stage-3 + Map Reduce + Alias -> Map Operator Tree: + /data/users/njain/hive1/hive/build/ql/tmp/249341865/269250345.10003 + Reduce Output Operator + key expressions: + expr: 0 + type: string + expr: 1 + type: string + sort order: ++ + Map-reduce partition columns: + expr: 0 + type: string + expr: 1 + type: string + tag: -1 + value expressions: + expr: 2 + type: bigint + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(DISTINCT KEY.1) + keys: + expr: KEY.0 + type: string + mode: partial2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + + Stage: Stage-4 + Map Reduce + Alias -> Map Operator Tree: + /data/users/njain/hive1/hive/build/ql/tmp/249341865/269250345.10004 + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: bigint + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: final + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: bigint + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest2 + + +0 1 +10 1 +100 1 +103 1 +104 1 +105 1 +11 1 +111 1 +113 1 +114 1 +116 1 +118 1 +119 1 +12 1 +120 1 +125 1 +126 1 +128 1 +129 1 +131 1 +133 1 +134 1 +136 1 +137 1 +138 1 +143 1 +145 1 +146 1 +149 1 +15 1 +150 1 +152 1 +153 1 +155 1 +156 1 +157 1 +158 1 +160 1 +162 1 +163 1 +164 1 +165 1 +166 1 +167 1 +168 1 +169 1 +17 1 +170 1 +172 1 +174 1 +175 1 +176 1 +177 1 +178 1 +179 1 +18 1 +180 1 +181 1 +183 1 +186 1 +187 1 +189 1 +19 1 +190 1 +191 1 +192 1 +193 1 +194 1 +195 1 +196 1 +197 1 +199 1 +2 1 +20 1 +200 1 +201 1 +202 1 +203 1 +205 1 +207 1 +208 1 +209 1 +213 1 +214 1 +216 1 +217 1 +218 1 +219 1 +221 1 +222 1 +223 1 +224 1 +226 1 +228 1 +229 1 +230 1 +233 1 +235 1 +237 1 +238 1 +239 1 +24 1 +241 1 +242 1 +244 1 +247 1 +248 1 +249 1 +252 1 +255 1 +256 1 +257 1 +258 1 +26 1 +260 1 +262 1 +263 1 +265 1 +266 1 +27 1 +272 1 +273 1 +274 1 +275 1 +277 1 +278 1 +28 1 +280 1 +281 1 +282 1 +283 1 +284 1 +285 1 +286 1 +287 1 +288 1 +289 1 +291 1 +292 1 +296 1 +298 1 +30 1 +302 1 +305 1 +306 1 +307 1 +308 1 +309 1 +310 1 +311 1 +315 1 +316 1 +317 1 +318 1 +321 1 +322 1 +323 1 +325 1 +327 1 +33 1 +331 1 +332 1 +333 1 +335 1 +336 1 +338 1 +339 1 +34 1 +341 1 +342 1 +344 1 +345 1 +348 1 +35 1 +351 1 +353 1 +356 1 +360 1 +362 1 +364 1 +365 1 +366 1 +367 1 +368 1 +369 1 +37 1 +373 1 +374 1 +375 1 +377 1 +378 1 +379 1 +382 1 +384 1 +386 1 +389 1 +392 1 +393 1 +394 1 +395 1 +396 1 +397 1 +399 1 +4 1 +400 1 +401 1 +402 1 +403 1 +404 1 +406 1 +407 1 +409 1 +41 1 +411 1 +413 1 +414 1 +417 1 +418 1 +419 1 +42 1 +421 1 +424 1 +427 1 +429 1 +43 1 +430 1 +431 1 +432 1 +435 1 +436 1 +437 1 +438 1 +439 1 +44 1 +443 1 +444 1 +446 1 +448 1 +449 1 +452 1 +453 1 +454 1 +455 1 +457 1 +458 1 +459 1 +460 1 +462 1 +463 1 +466 1 +467 1 +468 1 +469 1 +47 1 +470 1 +472 1 +475 1 +477 1 +478 1 +479 1 +480 1 +481 1 +482 1 +483 1 +484 1 +485 1 +487 1 +489 1 +490 1 +491 1 +492 1 +493 1 +494 1 +495 1 +496 1 +497 1 +498 1 +5 1 +51 1 +53 1 +54 1 +57 1 +58 1 +64 1 +65 1 +66 1 +67 1 +69 1 +70 1 +72 1 +74 1 +76 1 +77 1 +78 1 +8 1 +80 1 +82 1 +83 1 +84 1 +85 1 +86 1 +87 1 +9 1 +90 1 +92 1 +95 1 +96 1 +97 1 +98 1 +0 1 +10 1 +100 1 +103 1 +104 1 +105 1 +11 1 +111 1 +113 1 +114 1 +116 1 +118 1 +119 1 +12 1 +120 1 +125 1 +126 1 +128 1 +129 1 +131 1 +133 1 +134 1 +136 1 +137 1 +138 1 +143 1 +145 1 +146 1 +149 1 +15 1 +150 1 +152 1 +153 1 +155 1 +156 1 +157 1 +158 1 +160 1 +162 1 +163 1 +164 1 +165 1 +166 1 +167 1 +168 1 +169 1 +17 1 +170 1 +172 1 +174 1 +175 1 +176 1 +177 1 +178 1 +179 1 +18 1 +180 1 +181 1 +183 1 +186 1 +187 1 +189 1 +19 1 +190 1 +191 1 +192 1 +193 1 +194 1 +195 1 +196 1 +197 1 +199 1 +2 1 +20 1 +200 1 +201 1 +202 1 +203 1 +205 1 +207 1 +208 1 +209 1 +213 1 +214 1 +216 1 +217 1 +218 1 +219 1 +221 1 +222 1 +223 1 +224 1 +226 1 +228 1 +229 1 +230 1 +233 1 +235 1 +237 1 +238 1 +239 1 +24 1 +241 1 +242 1 +244 1 +247 1 +248 1 +249 1 +252 1 +255 1 +256 1 +257 1 +258 1 +26 1 +260 1 +262 1 +263 1 +265 1 +266 1 +27 1 +272 1 +273 1 +274 1 +275 1 +277 1 +278 1 +28 1 +280 1 +281 1 +282 1 +283 1 +284 1 +285 1 +286 1 +287 1 +288 1 +289 1 +291 1 +292 1 +296 1 +298 1 +30 1 +302 1 +305 1 +306 1 +307 1 +308 1 +309 1 +310 1 +311 1 +315 1 +316 1 +317 1 +318 1 +321 1 +322 1 +323 1 +325 1 +327 1 +33 1 +331 1 +332 1 +333 1 +335 1 +336 1 +338 1 +339 1 +34 1 +341 1 +342 1 +344 1 +345 1 +348 1 +35 1 +351 1 +353 1 +356 1 +360 1 +362 1 +364 1 +365 1 +366 1 +367 1 +368 1 +369 1 +37 1 +373 1 +374 1 +375 1 +377 1 +378 1 +379 1 +382 1 +384 1 +386 1 +389 1 +392 1 +393 1 +394 1 +395 1 +396 1 +397 1 +399 1 +4 1 +400 1 +401 1 +402 1 +403 1 +404 1 +406 1 +407 1 +409 1 +41 1 +411 1 +413 1 +414 1 +417 1 +418 1 +419 1 +42 1 +421 1 +424 1 +427 1 +429 1 +43 1 +430 1 +431 1 +432 1 +435 1 +436 1 +437 1 +438 1 +439 1 +44 1 +443 1 +444 1 +446 1 +448 1 +449 1 +452 1 +453 1 +454 1 +455 1 +457 1 +458 1 +459 1 +460 1 +462 1 +463 1 +466 1 +467 1 +468 1 +469 1 +47 1 +470 1 +472 1 +475 1 +477 1 +478 1 +479 1 +480 1 +481 1 +482 1 +483 1 +484 1 +485 1 +487 1 +489 1 +490 1 +491 1 +492 1 +493 1 +494 1 +495 1 +496 1 +497 1 +498 1 +5 1 +51 1 +53 1 +54 1 +57 1 +58 1 +64 1 +65 1 +66 1 +67 1 +69 1 +70 1 +72 1 +74 1 +76 1 +77 1 +78 1 +8 1 +80 1 +82 1 +83 1 +84 1 +85 1 +86 1 +87 1 +9 1 +90 1 +92 1 +95 1 +96 1 +97 1 +98 1 Index: ql/src/test/results/clientpositive/groupby4_map.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby4_map.q.out (revision 747596) +++ ql/src/test/results/clientpositive/groupby4_map.q.out (working copy) @@ -17,9 +17,6 @@ mode: hash Reduce Output Operator sort order: - Map-reduce partition columns: - expr: rand() - type: double tag: -1 value expressions: expr: 0 Index: ql/src/test/results/clientpositive/groupby8_map.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby8_map.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby8_map.q.out (revision 0) @@ -0,0 +1,749 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF SRC)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB DEST1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF SRC key)) (TOK_SELEXPR (TOK_FUNCTIONDI COUNT (TOK_FUNCTION SUBSTR (TOK_COLREF SRC value) 5)))) (TOK_GROUPBY (TOK_COLREF SRC key))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB DEST2)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF SRC key)) (TOK_SELEXPR (TOK_FUNCTIONDI COUNT (TOK_FUNCTION SUBSTR (TOK_COLREF SRC value) 5)))) (TOK_GROUPBY (TOK_COLREF SRC key)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1, Stage-2 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-1, Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Group By Operator + aggregations: + expr: count(DISTINCT substr(value, 5)) + keys: + expr: key + type: string + expr: substr(value, 5) + type: string + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + expr: 1 + type: string + sort order: ++ + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 2 + type: bigint + Group By Operator + aggregations: + expr: count(DISTINCT substr(value, 5)) + keys: + expr: key + type: string + expr: substr(value, 5) + type: string + mode: hash + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: final + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: bigint + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest2 + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + /data/users/njain/hive1/hive/build/ql/tmp/57407318/27303433.10002 + Reduce Output Operator + key expressions: + expr: 0 + type: string + expr: 1 + type: string + sort order: ++ + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 2 + type: bigint + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: final + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: bigint + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest2 + + +0 1 +10 1 +100 1 +103 1 +104 1 +105 1 +11 1 +111 1 +113 1 +114 1 +116 1 +118 1 +119 1 +12 1 +120 1 +125 1 +126 1 +128 1 +129 1 +131 1 +133 1 +134 1 +136 1 +137 1 +138 1 +143 1 +145 1 +146 1 +149 1 +15 1 +150 1 +152 1 +153 1 +155 1 +156 1 +157 1 +158 1 +160 1 +162 1 +163 1 +164 1 +165 1 +166 1 +167 1 +168 1 +169 1 +17 1 +170 1 +172 1 +174 1 +175 1 +176 1 +177 1 +178 1 +179 1 +18 1 +180 1 +181 1 +183 1 +186 1 +187 1 +189 1 +19 1 +190 1 +191 1 +192 1 +193 1 +194 1 +195 1 +196 1 +197 1 +199 1 +2 1 +20 1 +200 1 +201 1 +202 1 +203 1 +205 1 +207 1 +208 1 +209 1 +213 1 +214 1 +216 1 +217 1 +218 1 +219 1 +221 1 +222 1 +223 1 +224 1 +226 1 +228 1 +229 1 +230 1 +233 1 +235 1 +237 1 +238 1 +239 1 +24 1 +241 1 +242 1 +244 1 +247 1 +248 1 +249 1 +252 1 +255 1 +256 1 +257 1 +258 1 +26 1 +260 1 +262 1 +263 1 +265 1 +266 1 +27 1 +272 1 +273 1 +274 1 +275 1 +277 1 +278 1 +28 1 +280 1 +281 1 +282 1 +283 1 +284 1 +285 1 +286 1 +287 1 +288 1 +289 1 +291 1 +292 1 +296 1 +298 1 +30 1 +302 1 +305 1 +306 1 +307 1 +308 1 +309 1 +310 1 +311 1 +315 1 +316 1 +317 1 +318 1 +321 1 +322 1 +323 1 +325 1 +327 1 +33 1 +331 1 +332 1 +333 1 +335 1 +336 1 +338 1 +339 1 +34 1 +341 1 +342 1 +344 1 +345 1 +348 1 +35 1 +351 1 +353 1 +356 1 +360 1 +362 1 +364 1 +365 1 +366 1 +367 1 +368 1 +369 1 +37 1 +373 1 +374 1 +375 1 +377 1 +378 1 +379 1 +382 1 +384 1 +386 1 +389 1 +392 1 +393 1 +394 1 +395 1 +396 1 +397 1 +399 1 +4 1 +400 1 +401 1 +402 1 +403 1 +404 1 +406 1 +407 1 +409 1 +41 1 +411 1 +413 1 +414 1 +417 1 +418 1 +419 1 +42 1 +421 1 +424 1 +427 1 +429 1 +43 1 +430 1 +431 1 +432 1 +435 1 +436 1 +437 1 +438 1 +439 1 +44 1 +443 1 +444 1 +446 1 +448 1 +449 1 +452 1 +453 1 +454 1 +455 1 +457 1 +458 1 +459 1 +460 1 +462 1 +463 1 +466 1 +467 1 +468 1 +469 1 +47 1 +470 1 +472 1 +475 1 +477 1 +478 1 +479 1 +480 1 +481 1 +482 1 +483 1 +484 1 +485 1 +487 1 +489 1 +490 1 +491 1 +492 1 +493 1 +494 1 +495 1 +496 1 +497 1 +498 1 +5 1 +51 1 +53 1 +54 1 +57 1 +58 1 +64 1 +65 1 +66 1 +67 1 +69 1 +70 1 +72 1 +74 1 +76 1 +77 1 +78 1 +8 1 +80 1 +82 1 +83 1 +84 1 +85 1 +86 1 +87 1 +9 1 +90 1 +92 1 +95 1 +96 1 +97 1 +98 1 +0 1 +10 1 +100 1 +103 1 +104 1 +105 1 +11 1 +111 1 +113 1 +114 1 +116 1 +118 1 +119 1 +12 1 +120 1 +125 1 +126 1 +128 1 +129 1 +131 1 +133 1 +134 1 +136 1 +137 1 +138 1 +143 1 +145 1 +146 1 +149 1 +15 1 +150 1 +152 1 +153 1 +155 1 +156 1 +157 1 +158 1 +160 1 +162 1 +163 1 +164 1 +165 1 +166 1 +167 1 +168 1 +169 1 +17 1 +170 1 +172 1 +174 1 +175 1 +176 1 +177 1 +178 1 +179 1 +18 1 +180 1 +181 1 +183 1 +186 1 +187 1 +189 1 +19 1 +190 1 +191 1 +192 1 +193 1 +194 1 +195 1 +196 1 +197 1 +199 1 +2 1 +20 1 +200 1 +201 1 +202 1 +203 1 +205 1 +207 1 +208 1 +209 1 +213 1 +214 1 +216 1 +217 1 +218 1 +219 1 +221 1 +222 1 +223 1 +224 1 +226 1 +228 1 +229 1 +230 1 +233 1 +235 1 +237 1 +238 1 +239 1 +24 1 +241 1 +242 1 +244 1 +247 1 +248 1 +249 1 +252 1 +255 1 +256 1 +257 1 +258 1 +26 1 +260 1 +262 1 +263 1 +265 1 +266 1 +27 1 +272 1 +273 1 +274 1 +275 1 +277 1 +278 1 +28 1 +280 1 +281 1 +282 1 +283 1 +284 1 +285 1 +286 1 +287 1 +288 1 +289 1 +291 1 +292 1 +296 1 +298 1 +30 1 +302 1 +305 1 +306 1 +307 1 +308 1 +309 1 +310 1 +311 1 +315 1 +316 1 +317 1 +318 1 +321 1 +322 1 +323 1 +325 1 +327 1 +33 1 +331 1 +332 1 +333 1 +335 1 +336 1 +338 1 +339 1 +34 1 +341 1 +342 1 +344 1 +345 1 +348 1 +35 1 +351 1 +353 1 +356 1 +360 1 +362 1 +364 1 +365 1 +366 1 +367 1 +368 1 +369 1 +37 1 +373 1 +374 1 +375 1 +377 1 +378 1 +379 1 +382 1 +384 1 +386 1 +389 1 +392 1 +393 1 +394 1 +395 1 +396 1 +397 1 +399 1 +4 1 +400 1 +401 1 +402 1 +403 1 +404 1 +406 1 +407 1 +409 1 +41 1 +411 1 +413 1 +414 1 +417 1 +418 1 +419 1 +42 1 +421 1 +424 1 +427 1 +429 1 +43 1 +430 1 +431 1 +432 1 +435 1 +436 1 +437 1 +438 1 +439 1 +44 1 +443 1 +444 1 +446 1 +448 1 +449 1 +452 1 +453 1 +454 1 +455 1 +457 1 +458 1 +459 1 +460 1 +462 1 +463 1 +466 1 +467 1 +468 1 +469 1 +47 1 +470 1 +472 1 +475 1 +477 1 +478 1 +479 1 +480 1 +481 1 +482 1 +483 1 +484 1 +485 1 +487 1 +489 1 +490 1 +491 1 +492 1 +493 1 +494 1 +495 1 +496 1 +497 1 +498 1 +5 1 +51 1 +53 1 +54 1 +57 1 +58 1 +64 1 +65 1 +66 1 +67 1 +69 1 +70 1 +72 1 +74 1 +76 1 +77 1 +78 1 +8 1 +80 1 +82 1 +83 1 +84 1 +85 1 +86 1 +87 1 +9 1 +90 1 +92 1 +95 1 +96 1 +97 1 +98 1 Index: ql/src/test/results/clientpositive/groupby7_map_skew.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby7_map_skew.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby7_map_skew.q.out (revision 0) @@ -0,0 +1,805 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF SRC)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB DEST1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF SRC key)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION SUBSTR (TOK_COLREF SRC value) 5)))) (TOK_GROUPBY (TOK_COLREF SRC key))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB DEST2)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF SRC key)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION SUBSTR (TOK_COLREF SRC value) 5)))) (TOK_GROUPBY (TOK_COLREF SRC key)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2, Stage-4 + Stage-3 depends on stages: Stage-2 + Stage-4 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-2, Stage-4 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Group By Operator + aggregations: + expr: sum(UDFToDouble(substr(value, 5))) + keys: + expr: key + type: string + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: rand() + type: double + tag: -1 + value expressions: + expr: 1 + type: double + Group By Operator + aggregations: + expr: sum(UDFToDouble(substr(value, 5))) + keys: + expr: key + type: string + mode: hash + File Output Operator + compressed: true + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: partial2 + File Output Operator + compressed: true + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + /data/users/njain/hive1/hive/build/ql/tmp/116476002/106370095.10002 + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: double + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: final + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: double + File Output Operator + compressed: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest2 + + Stage: Stage-3 + Map Reduce + Alias -> Map Operator Tree: + /data/users/njain/hive1/hive/build/ql/tmp/116476002/106370095.10003 + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: rand() + type: double + tag: -1 + value expressions: + expr: 1 + type: double + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: partial2 + File Output Operator + compressed: true + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + + Stage: Stage-4 + Map Reduce + Alias -> Map Operator Tree: + /data/users/njain/hive1/hive/build/ql/tmp/116476002/106370095.10004 + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: double + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: final + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: double + File Output Operator + compressed: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest2 + + +0 0.0 +10 10.0 +100 200.0 +103 206.0 +104 208.0 +105 105.0 +11 11.0 +111 111.0 +113 226.0 +114 114.0 +116 116.0 +118 236.0 +119 357.0 +12 24.0 +120 240.0 +125 250.0 +126 126.0 +128 384.0 +129 258.0 +131 131.0 +133 133.0 +134 268.0 +136 136.0 +137 274.0 +138 552.0 +143 143.0 +145 145.0 +146 292.0 +149 298.0 +15 30.0 +150 150.0 +152 304.0 +153 153.0 +155 155.0 +156 156.0 +157 157.0 +158 158.0 +160 160.0 +162 162.0 +163 163.0 +164 328.0 +165 330.0 +166 166.0 +167 501.0 +168 168.0 +169 676.0 +17 17.0 +170 170.0 +172 344.0 +174 348.0 +175 350.0 +176 352.0 +177 177.0 +178 178.0 +179 358.0 +18 36.0 +180 180.0 +181 181.0 +183 183.0 +186 186.0 +187 561.0 +189 189.0 +19 19.0 +190 190.0 +191 382.0 +192 192.0 +193 579.0 +194 194.0 +195 390.0 +196 196.0 +197 394.0 +199 597.0 +2 2.0 +20 20.0 +200 400.0 +201 201.0 +202 202.0 +203 406.0 +205 410.0 +207 414.0 +208 624.0 +209 418.0 +213 426.0 +214 214.0 +216 432.0 +217 434.0 +218 218.0 +219 438.0 +221 442.0 +222 222.0 +223 446.0 +224 448.0 +226 226.0 +228 228.0 +229 458.0 +230 1150.0 +233 466.0 +235 235.0 +237 474.0 +238 476.0 +239 478.0 +24 48.0 +241 241.0 +242 484.0 +244 244.0 +247 247.0 +248 248.0 +249 249.0 +252 252.0 +255 510.0 +256 512.0 +257 257.0 +258 258.0 +26 52.0 +260 260.0 +262 262.0 +263 263.0 +265 530.0 +266 266.0 +27 27.0 +272 544.0 +273 819.0 +274 274.0 +275 275.0 +277 1108.0 +278 556.0 +28 28.0 +280 560.0 +281 562.0 +282 564.0 +283 283.0 +284 284.0 +285 285.0 +286 286.0 +287 287.0 +288 576.0 +289 289.0 +291 291.0 +292 292.0 +296 296.0 +298 894.0 +30 30.0 +302 302.0 +305 305.0 +306 306.0 +307 614.0 +308 308.0 +309 618.0 +310 310.0 +311 933.0 +315 315.0 +316 948.0 +317 634.0 +318 954.0 +321 642.0 +322 644.0 +323 323.0 +325 650.0 +327 981.0 +33 33.0 +331 662.0 +332 332.0 +333 666.0 +335 335.0 +336 336.0 +338 338.0 +339 339.0 +34 34.0 +341 341.0 +342 684.0 +344 688.0 +345 345.0 +348 1740.0 +35 105.0 +351 351.0 +353 706.0 +356 356.0 +360 360.0 +362 362.0 +364 364.0 +365 365.0 +366 366.0 +367 734.0 +368 368.0 +369 1107.0 +37 74.0 +373 373.0 +374 374.0 +375 375.0 +377 377.0 +378 378.0 +379 379.0 +382 764.0 +384 1152.0 +386 386.0 +389 389.0 +392 392.0 +393 393.0 +394 394.0 +395 790.0 +396 1188.0 +397 794.0 +399 798.0 +4 4.0 +400 400.0 +401 2005.0 +402 402.0 +403 1209.0 +404 808.0 +406 1624.0 +407 407.0 +409 1227.0 +41 41.0 +411 411.0 +413 826.0 +414 828.0 +417 1251.0 +418 418.0 +419 419.0 +42 84.0 +421 421.0 +424 848.0 +427 427.0 +429 858.0 +43 43.0 +430 1290.0 +431 1293.0 +432 432.0 +435 435.0 +436 436.0 +437 437.0 +438 1314.0 +439 878.0 +44 44.0 +443 443.0 +444 444.0 +446 446.0 +448 448.0 +449 449.0 +452 452.0 +453 453.0 +454 1362.0 +455 455.0 +457 457.0 +458 916.0 +459 918.0 +460 460.0 +462 924.0 +463 926.0 +466 1398.0 +467 467.0 +468 1872.0 +469 2345.0 +47 47.0 +470 470.0 +472 472.0 +475 475.0 +477 477.0 +478 956.0 +479 479.0 +480 1440.0 +481 481.0 +482 482.0 +483 483.0 +484 484.0 +485 485.0 +487 487.0 +489 1956.0 +490 490.0 +491 491.0 +492 984.0 +493 493.0 +494 494.0 +495 495.0 +496 496.0 +497 497.0 +498 1494.0 +5 15.0 +51 102.0 +53 53.0 +54 54.0 +57 57.0 +58 116.0 +64 64.0 +65 65.0 +66 66.0 +67 134.0 +69 69.0 +70 210.0 +72 144.0 +74 74.0 +76 152.0 +77 77.0 +78 78.0 +8 8.0 +80 80.0 +82 82.0 +83 166.0 +84 168.0 +85 85.0 +86 86.0 +87 87.0 +9 9.0 +90 270.0 +92 92.0 +95 190.0 +96 96.0 +97 194.0 +98 196.0 +0 0.0 +10 10.0 +100 200.0 +103 206.0 +104 208.0 +105 105.0 +11 11.0 +111 111.0 +113 226.0 +114 114.0 +116 116.0 +118 236.0 +119 357.0 +12 24.0 +120 240.0 +125 250.0 +126 126.0 +128 384.0 +129 258.0 +131 131.0 +133 133.0 +134 268.0 +136 136.0 +137 274.0 +138 552.0 +143 143.0 +145 145.0 +146 292.0 +149 298.0 +15 30.0 +150 150.0 +152 304.0 +153 153.0 +155 155.0 +156 156.0 +157 157.0 +158 158.0 +160 160.0 +162 162.0 +163 163.0 +164 328.0 +165 330.0 +166 166.0 +167 501.0 +168 168.0 +169 676.0 +17 17.0 +170 170.0 +172 344.0 +174 348.0 +175 350.0 +176 352.0 +177 177.0 +178 178.0 +179 358.0 +18 36.0 +180 180.0 +181 181.0 +183 183.0 +186 186.0 +187 561.0 +189 189.0 +19 19.0 +190 190.0 +191 382.0 +192 192.0 +193 579.0 +194 194.0 +195 390.0 +196 196.0 +197 394.0 +199 597.0 +2 2.0 +20 20.0 +200 400.0 +201 201.0 +202 202.0 +203 406.0 +205 410.0 +207 414.0 +208 624.0 +209 418.0 +213 426.0 +214 214.0 +216 432.0 +217 434.0 +218 218.0 +219 438.0 +221 442.0 +222 222.0 +223 446.0 +224 448.0 +226 226.0 +228 228.0 +229 458.0 +230 1150.0 +233 466.0 +235 235.0 +237 474.0 +238 476.0 +239 478.0 +24 48.0 +241 241.0 +242 484.0 +244 244.0 +247 247.0 +248 248.0 +249 249.0 +252 252.0 +255 510.0 +256 512.0 +257 257.0 +258 258.0 +26 52.0 +260 260.0 +262 262.0 +263 263.0 +265 530.0 +266 266.0 +27 27.0 +272 544.0 +273 819.0 +274 274.0 +275 275.0 +277 1108.0 +278 556.0 +28 28.0 +280 560.0 +281 562.0 +282 564.0 +283 283.0 +284 284.0 +285 285.0 +286 286.0 +287 287.0 +288 576.0 +289 289.0 +291 291.0 +292 292.0 +296 296.0 +298 894.0 +30 30.0 +302 302.0 +305 305.0 +306 306.0 +307 614.0 +308 308.0 +309 618.0 +310 310.0 +311 933.0 +315 315.0 +316 948.0 +317 634.0 +318 954.0 +321 642.0 +322 644.0 +323 323.0 +325 650.0 +327 981.0 +33 33.0 +331 662.0 +332 332.0 +333 666.0 +335 335.0 +336 336.0 +338 338.0 +339 339.0 +34 34.0 +341 341.0 +342 684.0 +344 688.0 +345 345.0 +348 1740.0 +35 105.0 +351 351.0 +353 706.0 +356 356.0 +360 360.0 +362 362.0 +364 364.0 +365 365.0 +366 366.0 +367 734.0 +368 368.0 +369 1107.0 +37 74.0 +373 373.0 +374 374.0 +375 375.0 +377 377.0 +378 378.0 +379 379.0 +382 764.0 +384 1152.0 +386 386.0 +389 389.0 +392 392.0 +393 393.0 +394 394.0 +395 790.0 +396 1188.0 +397 794.0 +399 798.0 +4 4.0 +400 400.0 +401 2005.0 +402 402.0 +403 1209.0 +404 808.0 +406 1624.0 +407 407.0 +409 1227.0 +41 41.0 +411 411.0 +413 826.0 +414 828.0 +417 1251.0 +418 418.0 +419 419.0 +42 84.0 +421 421.0 +424 848.0 +427 427.0 +429 858.0 +43 43.0 +430 1290.0 +431 1293.0 +432 432.0 +435 435.0 +436 436.0 +437 437.0 +438 1314.0 +439 878.0 +44 44.0 +443 443.0 +444 444.0 +446 446.0 +448 448.0 +449 449.0 +452 452.0 +453 453.0 +454 1362.0 +455 455.0 +457 457.0 +458 916.0 +459 918.0 +460 460.0 +462 924.0 +463 926.0 +466 1398.0 +467 467.0 +468 1872.0 +469 2345.0 +47 47.0 +470 470.0 +472 472.0 +475 475.0 +477 477.0 +478 956.0 +479 479.0 +480 1440.0 +481 481.0 +482 482.0 +483 483.0 +484 484.0 +485 485.0 +487 487.0 +489 1956.0 +490 490.0 +491 491.0 +492 984.0 +493 493.0 +494 494.0 +495 495.0 +496 496.0 +497 497.0 +498 1494.0 +5 15.0 +51 102.0 +53 53.0 +54 54.0 +57 57.0 +58 116.0 +64 64.0 +65 65.0 +66 66.0 +67 134.0 +69 69.0 +70 210.0 +72 144.0 +74 74.0 +76 152.0 +77 77.0 +78 78.0 +8 8.0 +80 80.0 +82 82.0 +83 166.0 +84 168.0 +85 85.0 +86 86.0 +87 87.0 +9 9.0 +90 270.0 +92 92.0 +95 190.0 +96 96.0 +97 194.0 +98 196.0 Index: ql/src/test/results/clientpositive/groupby8_noskew.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby8_noskew.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby8_noskew.q.out (revision 0) @@ -0,0 +1,725 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF SRC)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB DEST1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF SRC key)) (TOK_SELEXPR (TOK_FUNCTIONDI COUNT (TOK_FUNCTION SUBSTR (TOK_COLREF SRC value) 5)))) (TOK_GROUPBY (TOK_COLREF SRC key))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB DEST2)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF SRC key)) (TOK_SELEXPR (TOK_FUNCTIONDI COUNT (TOK_FUNCTION SUBSTR (TOK_COLREF SRC value) 5)))) (TOK_GROUPBY (TOK_COLREF SRC key)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1, Stage-2 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-1, Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Reduce Output Operator + key expressions: + expr: key + type: string + expr: substr(value, 5) + type: string + sort order: ++ + Map-reduce partition columns: + expr: key + type: string + tag: -1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(DISTINCT KEY.1) + keys: + expr: KEY.0 + type: string + mode: complete + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: bigint + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest2 + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + /data/users/njain/hive1/hive/build/ql/tmp/195863835/4729299.10002 + Reduce Output Operator + key expressions: + expr: key + type: string + expr: substr(value, 5) + type: string + sort order: ++ + Map-reduce partition columns: + expr: key + type: string + tag: -1 + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(DISTINCT KEY.1) + keys: + expr: KEY.0 + type: string + mode: complete + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: bigint + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest2 + + +0 1 +10 1 +100 1 +103 1 +104 1 +105 1 +11 1 +111 1 +113 1 +114 1 +116 1 +118 1 +119 1 +12 1 +120 1 +125 1 +126 1 +128 1 +129 1 +131 1 +133 1 +134 1 +136 1 +137 1 +138 1 +143 1 +145 1 +146 1 +149 1 +15 1 +150 1 +152 1 +153 1 +155 1 +156 1 +157 1 +158 1 +160 1 +162 1 +163 1 +164 1 +165 1 +166 1 +167 1 +168 1 +169 1 +17 1 +170 1 +172 1 +174 1 +175 1 +176 1 +177 1 +178 1 +179 1 +18 1 +180 1 +181 1 +183 1 +186 1 +187 1 +189 1 +19 1 +190 1 +191 1 +192 1 +193 1 +194 1 +195 1 +196 1 +197 1 +199 1 +2 1 +20 1 +200 1 +201 1 +202 1 +203 1 +205 1 +207 1 +208 1 +209 1 +213 1 +214 1 +216 1 +217 1 +218 1 +219 1 +221 1 +222 1 +223 1 +224 1 +226 1 +228 1 +229 1 +230 1 +233 1 +235 1 +237 1 +238 1 +239 1 +24 1 +241 1 +242 1 +244 1 +247 1 +248 1 +249 1 +252 1 +255 1 +256 1 +257 1 +258 1 +26 1 +260 1 +262 1 +263 1 +265 1 +266 1 +27 1 +272 1 +273 1 +274 1 +275 1 +277 1 +278 1 +28 1 +280 1 +281 1 +282 1 +283 1 +284 1 +285 1 +286 1 +287 1 +288 1 +289 1 +291 1 +292 1 +296 1 +298 1 +30 1 +302 1 +305 1 +306 1 +307 1 +308 1 +309 1 +310 1 +311 1 +315 1 +316 1 +317 1 +318 1 +321 1 +322 1 +323 1 +325 1 +327 1 +33 1 +331 1 +332 1 +333 1 +335 1 +336 1 +338 1 +339 1 +34 1 +341 1 +342 1 +344 1 +345 1 +348 1 +35 1 +351 1 +353 1 +356 1 +360 1 +362 1 +364 1 +365 1 +366 1 +367 1 +368 1 +369 1 +37 1 +373 1 +374 1 +375 1 +377 1 +378 1 +379 1 +382 1 +384 1 +386 1 +389 1 +392 1 +393 1 +394 1 +395 1 +396 1 +397 1 +399 1 +4 1 +400 1 +401 1 +402 1 +403 1 +404 1 +406 1 +407 1 +409 1 +41 1 +411 1 +413 1 +414 1 +417 1 +418 1 +419 1 +42 1 +421 1 +424 1 +427 1 +429 1 +43 1 +430 1 +431 1 +432 1 +435 1 +436 1 +437 1 +438 1 +439 1 +44 1 +443 1 +444 1 +446 1 +448 1 +449 1 +452 1 +453 1 +454 1 +455 1 +457 1 +458 1 +459 1 +460 1 +462 1 +463 1 +466 1 +467 1 +468 1 +469 1 +47 1 +470 1 +472 1 +475 1 +477 1 +478 1 +479 1 +480 1 +481 1 +482 1 +483 1 +484 1 +485 1 +487 1 +489 1 +490 1 +491 1 +492 1 +493 1 +494 1 +495 1 +496 1 +497 1 +498 1 +5 1 +51 1 +53 1 +54 1 +57 1 +58 1 +64 1 +65 1 +66 1 +67 1 +69 1 +70 1 +72 1 +74 1 +76 1 +77 1 +78 1 +8 1 +80 1 +82 1 +83 1 +84 1 +85 1 +86 1 +87 1 +9 1 +90 1 +92 1 +95 1 +96 1 +97 1 +98 1 +0 1 +10 1 +100 1 +103 1 +104 1 +105 1 +11 1 +111 1 +113 1 +114 1 +116 1 +118 1 +119 1 +12 1 +120 1 +125 1 +126 1 +128 1 +129 1 +131 1 +133 1 +134 1 +136 1 +137 1 +138 1 +143 1 +145 1 +146 1 +149 1 +15 1 +150 1 +152 1 +153 1 +155 1 +156 1 +157 1 +158 1 +160 1 +162 1 +163 1 +164 1 +165 1 +166 1 +167 1 +168 1 +169 1 +17 1 +170 1 +172 1 +174 1 +175 1 +176 1 +177 1 +178 1 +179 1 +18 1 +180 1 +181 1 +183 1 +186 1 +187 1 +189 1 +19 1 +190 1 +191 1 +192 1 +193 1 +194 1 +195 1 +196 1 +197 1 +199 1 +2 1 +20 1 +200 1 +201 1 +202 1 +203 1 +205 1 +207 1 +208 1 +209 1 +213 1 +214 1 +216 1 +217 1 +218 1 +219 1 +221 1 +222 1 +223 1 +224 1 +226 1 +228 1 +229 1 +230 1 +233 1 +235 1 +237 1 +238 1 +239 1 +24 1 +241 1 +242 1 +244 1 +247 1 +248 1 +249 1 +252 1 +255 1 +256 1 +257 1 +258 1 +26 1 +260 1 +262 1 +263 1 +265 1 +266 1 +27 1 +272 1 +273 1 +274 1 +275 1 +277 1 +278 1 +28 1 +280 1 +281 1 +282 1 +283 1 +284 1 +285 1 +286 1 +287 1 +288 1 +289 1 +291 1 +292 1 +296 1 +298 1 +30 1 +302 1 +305 1 +306 1 +307 1 +308 1 +309 1 +310 1 +311 1 +315 1 +316 1 +317 1 +318 1 +321 1 +322 1 +323 1 +325 1 +327 1 +33 1 +331 1 +332 1 +333 1 +335 1 +336 1 +338 1 +339 1 +34 1 +341 1 +342 1 +344 1 +345 1 +348 1 +35 1 +351 1 +353 1 +356 1 +360 1 +362 1 +364 1 +365 1 +366 1 +367 1 +368 1 +369 1 +37 1 +373 1 +374 1 +375 1 +377 1 +378 1 +379 1 +382 1 +384 1 +386 1 +389 1 +392 1 +393 1 +394 1 +395 1 +396 1 +397 1 +399 1 +4 1 +400 1 +401 1 +402 1 +403 1 +404 1 +406 1 +407 1 +409 1 +41 1 +411 1 +413 1 +414 1 +417 1 +418 1 +419 1 +42 1 +421 1 +424 1 +427 1 +429 1 +43 1 +430 1 +431 1 +432 1 +435 1 +436 1 +437 1 +438 1 +439 1 +44 1 +443 1 +444 1 +446 1 +448 1 +449 1 +452 1 +453 1 +454 1 +455 1 +457 1 +458 1 +459 1 +460 1 +462 1 +463 1 +466 1 +467 1 +468 1 +469 1 +47 1 +470 1 +472 1 +475 1 +477 1 +478 1 +479 1 +480 1 +481 1 +482 1 +483 1 +484 1 +485 1 +487 1 +489 1 +490 1 +491 1 +492 1 +493 1 +494 1 +495 1 +496 1 +497 1 +498 1 +5 1 +51 1 +53 1 +54 1 +57 1 +58 1 +64 1 +65 1 +66 1 +67 1 +69 1 +70 1 +72 1 +74 1 +76 1 +77 1 +78 1 +8 1 +80 1 +82 1 +83 1 +84 1 +85 1 +86 1 +87 1 +9 1 +90 1 +92 1 +95 1 +96 1 +97 1 +98 1 Index: ql/src/test/results/clientpositive/groupby5_noskew.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby5_noskew.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby5_noskew.q.out (revision 0) @@ -0,0 +1,366 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src key)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION substr (TOK_COLREF src value) 5)))) (TOK_GROUPBY (TOK_COLREF src key)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: -1 + value expressions: + expr: substr(value, 5) + type: string + Reduce Operator Tree: + Group By Operator + aggregations: + expr: sum(UDFToDouble(VALUE.0)) + keys: + expr: KEY.0 + type: string + mode: complete + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: double + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + +0 0.0 +10 10.0 +100 200.0 +103 206.0 +104 208.0 +105 105.0 +11 11.0 +111 111.0 +113 226.0 +114 114.0 +116 116.0 +118 236.0 +119 357.0 +12 24.0 +120 240.0 +125 250.0 +126 126.0 +128 384.0 +129 258.0 +131 131.0 +133 133.0 +134 268.0 +136 136.0 +137 274.0 +138 552.0 +143 143.0 +145 145.0 +146 292.0 +149 298.0 +15 30.0 +150 150.0 +152 304.0 +153 153.0 +155 155.0 +156 156.0 +157 157.0 +158 158.0 +160 160.0 +162 162.0 +163 163.0 +164 328.0 +165 330.0 +166 166.0 +167 501.0 +168 168.0 +169 676.0 +17 17.0 +170 170.0 +172 344.0 +174 348.0 +175 350.0 +176 352.0 +177 177.0 +178 178.0 +179 358.0 +18 36.0 +180 180.0 +181 181.0 +183 183.0 +186 186.0 +187 561.0 +189 189.0 +19 19.0 +190 190.0 +191 382.0 +192 192.0 +193 579.0 +194 194.0 +195 390.0 +196 196.0 +197 394.0 +199 597.0 +2 2.0 +20 20.0 +200 400.0 +201 201.0 +202 202.0 +203 406.0 +205 410.0 +207 414.0 +208 624.0 +209 418.0 +213 426.0 +214 214.0 +216 432.0 +217 434.0 +218 218.0 +219 438.0 +221 442.0 +222 222.0 +223 446.0 +224 448.0 +226 226.0 +228 228.0 +229 458.0 +230 1150.0 +233 466.0 +235 235.0 +237 474.0 +238 476.0 +239 478.0 +24 48.0 +241 241.0 +242 484.0 +244 244.0 +247 247.0 +248 248.0 +249 249.0 +252 252.0 +255 510.0 +256 512.0 +257 257.0 +258 258.0 +26 52.0 +260 260.0 +262 262.0 +263 263.0 +265 530.0 +266 266.0 +27 27.0 +272 544.0 +273 819.0 +274 274.0 +275 275.0 +277 1108.0 +278 556.0 +28 28.0 +280 560.0 +281 562.0 +282 564.0 +283 283.0 +284 284.0 +285 285.0 +286 286.0 +287 287.0 +288 576.0 +289 289.0 +291 291.0 +292 292.0 +296 296.0 +298 894.0 +30 30.0 +302 302.0 +305 305.0 +306 306.0 +307 614.0 +308 308.0 +309 618.0 +310 310.0 +311 933.0 +315 315.0 +316 948.0 +317 634.0 +318 954.0 +321 642.0 +322 644.0 +323 323.0 +325 650.0 +327 981.0 +33 33.0 +331 662.0 +332 332.0 +333 666.0 +335 335.0 +336 336.0 +338 338.0 +339 339.0 +34 34.0 +341 341.0 +342 684.0 +344 688.0 +345 345.0 +348 1740.0 +35 105.0 +351 351.0 +353 706.0 +356 356.0 +360 360.0 +362 362.0 +364 364.0 +365 365.0 +366 366.0 +367 734.0 +368 368.0 +369 1107.0 +37 74.0 +373 373.0 +374 374.0 +375 375.0 +377 377.0 +378 378.0 +379 379.0 +382 764.0 +384 1152.0 +386 386.0 +389 389.0 +392 392.0 +393 393.0 +394 394.0 +395 790.0 +396 1188.0 +397 794.0 +399 798.0 +4 4.0 +400 400.0 +401 2005.0 +402 402.0 +403 1209.0 +404 808.0 +406 1624.0 +407 407.0 +409 1227.0 +41 41.0 +411 411.0 +413 826.0 +414 828.0 +417 1251.0 +418 418.0 +419 419.0 +42 84.0 +421 421.0 +424 848.0 +427 427.0 +429 858.0 +43 43.0 +430 1290.0 +431 1293.0 +432 432.0 +435 435.0 +436 436.0 +437 437.0 +438 1314.0 +439 878.0 +44 44.0 +443 443.0 +444 444.0 +446 446.0 +448 448.0 +449 449.0 +452 452.0 +453 453.0 +454 1362.0 +455 455.0 +457 457.0 +458 916.0 +459 918.0 +460 460.0 +462 924.0 +463 926.0 +466 1398.0 +467 467.0 +468 1872.0 +469 2345.0 +47 47.0 +470 470.0 +472 472.0 +475 475.0 +477 477.0 +478 956.0 +479 479.0 +480 1440.0 +481 481.0 +482 482.0 +483 483.0 +484 484.0 +485 485.0 +487 487.0 +489 1956.0 +490 490.0 +491 491.0 +492 984.0 +493 493.0 +494 494.0 +495 495.0 +496 496.0 +497 497.0 +498 1494.0 +5 15.0 +51 102.0 +53 53.0 +54 54.0 +57 57.0 +58 116.0 +64 64.0 +65 65.0 +66 66.0 +67 134.0 +69 69.0 +70 210.0 +72 144.0 +74 74.0 +76 152.0 +77 77.0 +78 78.0 +8 8.0 +80 80.0 +82 82.0 +83 166.0 +84 168.0 +85 85.0 +86 86.0 +87 87.0 +9 9.0 +90 270.0 +92 92.0 +95 190.0 +96 96.0 +97 194.0 +98 196.0 Index: ql/src/test/results/clientpositive/groupby6_map_skew.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby6_map_skew.q.out (revision 0) +++ ql/src/test/results/clientpositive/groupby6_map_skew.q.out (revision 0) @@ -0,0 +1,96 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECTDI (TOK_SELEXPR (TOK_FUNCTION substr (TOK_COLREF src value) 5 1))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Select Operator + expressions: + expr: value + type: string + Group By Operator + keys: + expr: substr(0, 5, 1) + type: string + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: rand() + type: double + tag: -1 + Reduce Operator Tree: + Group By Operator + keys: + expr: KEY.0 + type: string + mode: partial2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + /data/users/njain/hive1/hive/build/ql/tmp/49768590/975099101.10001 + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + Reduce Operator Tree: + Group By Operator + keys: + expr: KEY.0 + type: string + mode: final + Select Operator + expressions: + expr: 0 + type: string + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: dest1 + + +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 Index: ql/src/test/results/clientpositive/groupby1_map.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby1_map.q.out (revision 747596) +++ ql/src/test/results/clientpositive/groupby1_map.q.out (working copy) @@ -3,8 +3,7 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 @@ -24,8 +23,8 @@ type: string sort order: + Map-reduce partition columns: - expr: rand() - type: double + expr: 0 + type: string tag: -1 value expressions: expr: 1 @@ -37,37 +36,6 @@ keys: expr: KEY.0 type: string - mode: partial2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.mapred.SequenceFileOutputFormat - name: binary_table - - Stage: Stage-2 - Map Reduce - Alias -> Map Operator Tree: - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/580217776/833275504.10001 - Reduce Output Operator - key expressions: - expr: 0 - type: string - sort order: + - Map-reduce partition columns: - expr: 0 - type: string - tag: -1 - value expressions: - expr: 1 - type: double - Reduce Operator Tree: - Group By Operator - aggregations: - expr: sum(VALUE.0) - keys: - expr: KEY.0 - type: string mode: final Select Operator expressions: Index: ql/src/test/results/clientpositive/groupby1_limit.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby1_limit.q.out (revision 747596) +++ ql/src/test/results/clientpositive/groupby1_limit.q.out (working copy) @@ -4,60 +4,35 @@ STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 - Stage-3 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 Map Reduce Alias -> Map Operator Tree: src - Reduce Output Operator - key expressions: + Group By Operator + aggregations: + expr: sum(UDFToDouble(substr(value, 5))) + keys: expr: key type: string - sort order: + - Map-reduce partition columns: - expr: rand() - type: double - tag: -1 - value expressions: - expr: substr(value, 5) - type: string + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + sort order: + + Map-reduce partition columns: + expr: 0 + type: string + tag: -1 + value expressions: + expr: 1 + type: double Reduce Operator Tree: Group By Operator aggregations: - expr: sum(UDFToDouble(VALUE.0)) - keys: - expr: KEY.0 - type: string - mode: partial1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.mapred.SequenceFileOutputFormat - name: binary_table - - Stage: Stage-2 - Map Reduce - Alias -> Map Operator Tree: - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/23529028/381361795.10001 - Reduce Output Operator - key expressions: - expr: 0 - type: string - sort order: + - Map-reduce partition columns: - expr: 0 - type: string - tag: -1 - value expressions: - expr: 1 - type: double - Reduce Operator Tree: - Group By Operator - aggregations: expr: sum(VALUE.0) keys: expr: KEY.0 @@ -77,10 +52,10 @@ output format: org.apache.hadoop.mapred.SequenceFileOutputFormat name: binary_table - Stage: Stage-3 + Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/23529028/381361795.10002 + /data/users/njain/hive1/hive/build/ql/tmp/73642028/101260284.10001 Reduce Output Operator sort order: tag: -1 Index: ql/src/test/results/clientpositive/groupby5_map.q.out =================================================================== --- ql/src/test/results/clientpositive/groupby5_map.q.out (revision 747596) +++ ql/src/test/results/clientpositive/groupby5_map.q.out (working copy) @@ -20,9 +20,6 @@ mode: hash Reduce Output Operator sort order: - Map-reduce partition columns: - expr: rand() - type: double tag: -1 value expressions: expr: 0 Index: ql/src/test/results/compiler/plan/join2.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join2.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/join2.q.xml (working copy) @@ -35,7 +35,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/1372738136.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/11202080.10000.insclause-0 @@ -88,7 +88,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -273,6 +273,9 @@ + + 352 + @@ -312,6 +315,9 @@ + + 336 + @@ -540,6 +546,9 @@ + + 351 + @@ -551,7 +560,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/1398982247/30641865.10001 + /data/users/njain/hive1/hive/build/ql/tmp/234612036/74740411.10001 @@ -584,6 +593,9 @@ + + 356 + @@ -672,6 +684,9 @@ + + 350 + @@ -776,6 +791,9 @@ + + 348 + @@ -805,6 +823,9 @@ + + 347 + @@ -816,6 +837,9 @@ + + 337 + @@ -995,6 +1019,9 @@ + + 349 + @@ -1024,6 +1051,9 @@ + + 346 + @@ -1035,6 +1065,9 @@ + + 335 + @@ -1162,7 +1195,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/1398982247/30641865.10001 + /data/users/njain/hive1/hive/build/ql/tmp/234612036/74740411.10001 $INTNAME @@ -1170,7 +1203,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src src3 @@ -1182,7 +1215,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/1398982247/30641865.10001 + /data/users/njain/hive1/hive/build/ql/tmp/234612036/74740411.10001 @@ -1190,7 +1223,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -1246,7 +1279,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -1272,13 +1305,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/1372738136.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/11202080.10000.insclause-0 + + 355 + @@ -1346,6 +1382,9 @@ + + 354 + @@ -1433,6 +1472,9 @@ + + 353 + @@ -1538,7 +1580,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src src2 @@ -1553,7 +1595,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -1609,7 +1651,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/input2.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input2.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/input2.q.xml (working copy) @@ -31,7 +31,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/658702358/146689636.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1291902977/74890834.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -104,7 +104,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/658702358/146689636.10001.insclause-1 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1291902977/74890834.10001.insclause-1 @@ -157,7 +157,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest2 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest2 @@ -186,7 +186,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/658702358/146689636.10002.insclause-2 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1291902977/74890834.10002.insclause-2 @@ -243,7 +243,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest3 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest3 @@ -287,13 +287,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/658702358/146689636.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1291902977/74890834.10000.insclause-0 + + 130 + @@ -368,6 +371,9 @@ + + 129 + @@ -497,6 +503,9 @@ + + 128 + @@ -547,13 +556,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/658702358/146689636.10001.insclause-1 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1291902977/74890834.10001.insclause-1 + + 133 + @@ -621,6 +633,9 @@ + + 132 + @@ -860,6 +875,9 @@ + + 131 + @@ -889,13 +907,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/658702358/146689636.10002.insclause-2 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1291902977/74890834.10002.insclause-2 + + 136 + @@ -963,6 +984,9 @@ + + 135 + @@ -1080,6 +1104,9 @@ + + 134 + @@ -1098,6 +1125,9 @@ + + 127 + @@ -1112,7 +1142,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src src @@ -1124,7 +1154,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -1180,7 +1210,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/join3.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join3.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/join3.q.xml (working copy) @@ -31,7 +31,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/31305467.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/471309597.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -228,6 +228,9 @@ + + 383 + @@ -275,6 +278,9 @@ + + 380 + @@ -304,6 +310,9 @@ + + 371 + @@ -454,6 +463,9 @@ + + 384 + @@ -493,6 +505,9 @@ + + 372 + @@ -634,6 +649,9 @@ + + 382 + @@ -681,6 +699,9 @@ + + 381 + @@ -710,6 +731,9 @@ + + 373 + @@ -754,7 +778,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src src2 @@ -772,7 +796,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -828,7 +852,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -854,13 +878,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/31305467.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/471309597.10000.insclause-0 + + 387 + @@ -928,6 +955,9 @@ + + 386 + @@ -1027,6 +1057,9 @@ + + 385 + Index: ql/src/test/results/compiler/plan/input3.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input3.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/input3.q.xml (working copy) @@ -28,7 +28,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/374455279/297135950.10003.insclause-3 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/528453469/1179321710.10003.insclause-3 ../../../../build/contrib/hive/ql/test/data/warehouse/dest4.out @@ -48,7 +48,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/374455279/297135950.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/528453469/1179321710.10000.insclause-0 @@ -101,7 +101,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -121,7 +121,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/374455279/297135950.10001.insclause-1 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/528453469/1179321710.10001.insclause-1 @@ -174,7 +174,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest2 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest2 @@ -203,7 +203,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/374455279/297135950.10002.insclause-2 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/528453469/1179321710.10002.insclause-2 @@ -260,7 +260,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest3 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest3 @@ -304,13 +304,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/374455279/297135950.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/528453469/1179321710.10000.insclause-0 + + 175 + @@ -385,6 +388,9 @@ + + 174 + @@ -514,6 +520,9 @@ + + 173 + @@ -564,13 +573,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/374455279/297135950.10001.insclause-1 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/528453469/1179321710.10001.insclause-1 + + 178 + @@ -638,6 +650,9 @@ + + 177 + @@ -877,6 +892,9 @@ + + 176 + @@ -906,13 +924,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/374455279/297135950.10002.insclause-2 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/528453469/1179321710.10002.insclause-2 + + 181 + @@ -980,6 +1001,9 @@ + + 180 + @@ -1219,6 +1243,9 @@ + + 179 + @@ -1248,7 +1275,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/374455279/297135950.10003.insclause-3 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/528453469/1179321710.10003.insclause-3 @@ -1277,6 +1304,9 @@ + + 184 + @@ -1324,6 +1354,9 @@ + + 183 + @@ -1441,6 +1474,9 @@ + + 182 + @@ -1459,6 +1495,9 @@ + + 172 + @@ -1473,7 +1512,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src src @@ -1485,7 +1524,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -1541,7 +1580,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/join4.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join4.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/join4.q.xml (working copy) @@ -140,6 +140,9 @@ + + 406 + @@ -207,6 +210,9 @@ + + 401 + @@ -479,6 +485,9 @@ + + 400 + @@ -518,6 +527,9 @@ + + 399 + @@ -652,6 +664,9 @@ + + 405 + @@ -719,6 +734,9 @@ + + 404 + @@ -979,6 +997,9 @@ + + 403 + @@ -1018,6 +1039,9 @@ + + 402 + @@ -1041,7 +1065,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src c:a:src1 @@ -1056,7 +1080,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -1112,7 +1136,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -1142,7 +1166,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/919060805/17880936.10001.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/58057286/297438101.10001.insclause-0 @@ -1171,6 +1195,9 @@ + + 410 + @@ -1278,6 +1305,9 @@ + + 409 + @@ -1344,6 +1374,9 @@ + + 408 + @@ -1475,6 +1508,9 @@ + + 407 + Index: ql/src/test/results/compiler/plan/input4.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input4.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/input4.q.xml (working copy) @@ -31,7 +31,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/881929024/614130622.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1547142572/1216232845.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -253,6 +253,9 @@ + + 201 + @@ -349,6 +352,9 @@ + + 200 + @@ -395,6 +401,9 @@ + + 199 + @@ -434,6 +443,9 @@ + + 198 + @@ -475,7 +487,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src tmap:src @@ -487,7 +499,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -543,7 +555,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -573,13 +585,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/881929024/614130622.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1547142572/1216232845.10000.insclause-0 + + 205 + @@ -647,6 +662,9 @@ + + 204 + @@ -776,6 +794,9 @@ + + 203 + @@ -829,6 +850,9 @@ + + 202 + Index: ql/src/test/results/compiler/plan/join5.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join5.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/join5.q.xml (working copy) @@ -140,6 +140,9 @@ + + 430 + @@ -207,6 +210,9 @@ + + 425 + @@ -479,6 +485,9 @@ + + 424 + @@ -518,6 +527,9 @@ + + 423 + @@ -652,6 +664,9 @@ + + 429 + @@ -719,6 +734,9 @@ + + 428 + @@ -979,6 +997,9 @@ + + 427 + @@ -1018,6 +1039,9 @@ + + 426 + @@ -1041,7 +1065,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src c:a:src1 @@ -1056,7 +1080,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -1112,7 +1136,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -1142,7 +1166,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/706416180/344763845.10001.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/155967095/309101621.10001.insclause-0 @@ -1171,6 +1195,9 @@ + + 434 + @@ -1278,6 +1305,9 @@ + + 433 + @@ -1344,6 +1374,9 @@ + + 432 + @@ -1475,6 +1508,9 @@ + + 431 + Index: ql/src/test/results/compiler/plan/input5.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input5.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/input5.q.xml (working copy) @@ -31,7 +31,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/189543713.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1122950416.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -257,6 +257,9 @@ + + 224 + @@ -353,6 +356,9 @@ + + 223 + @@ -411,6 +417,9 @@ + + 222 + @@ -478,6 +487,9 @@ + + 221 + @@ -517,6 +529,9 @@ + + 214 + @@ -609,7 +624,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src_thrift + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src_thrift tmap:src_thrift @@ -621,7 +636,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src_thrift + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src_thrift @@ -681,7 +696,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src_thrift + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src_thrift @@ -707,13 +722,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/189543713.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1122950416.10000.insclause-0 + + 227 + @@ -781,6 +799,9 @@ + + 226 + @@ -813,6 +834,9 @@ + + 225 + Index: ql/src/test/results/compiler/plan/join6.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join6.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/join6.q.xml (working copy) @@ -140,6 +140,9 @@ + + 454 + @@ -207,6 +210,9 @@ + + 449 + @@ -479,6 +485,9 @@ + + 448 + @@ -518,6 +527,9 @@ + + 447 + @@ -652,6 +664,9 @@ + + 453 + @@ -719,6 +734,9 @@ + + 452 + @@ -979,6 +997,9 @@ + + 451 + @@ -1018,6 +1039,9 @@ + + 450 + @@ -1041,7 +1065,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src c:a:src1 @@ -1056,7 +1080,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -1112,7 +1136,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -1142,7 +1166,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/554358737/14067720.10001.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/510290766/36796370.10001.insclause-0 @@ -1171,6 +1195,9 @@ + + 458 + @@ -1278,6 +1305,9 @@ + + 457 + @@ -1344,6 +1374,9 @@ + + 456 + @@ -1475,6 +1508,9 @@ + + 455 + Index: ql/src/test/results/compiler/plan/input_testxpath2.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input_testxpath2.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/input_testxpath2.q.xml (working copy) @@ -30,7 +30,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/513871881.10001.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/586486000.10001.insclause-0 @@ -59,6 +59,9 @@ + + 308 + @@ -241,6 +244,9 @@ + + 307 + @@ -386,6 +392,9 @@ + + 306 + @@ -473,6 +482,9 @@ + + 305 + @@ -491,6 +503,9 @@ + + 301 + @@ -570,7 +585,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src_thrift + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src_thrift src_thrift @@ -582,7 +597,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src_thrift + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src_thrift @@ -642,7 +657,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src_thrift + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src_thrift Index: ql/src/test/results/compiler/plan/input6.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input6.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/input6.q.xml (working copy) @@ -31,7 +31,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/267308598/510367048.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/974064460/650130902.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -128,13 +128,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/267308598/510367048.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/974064460/650130902.10000.insclause-0 + + 239 + @@ -206,6 +209,9 @@ + + 238 + @@ -266,6 +272,9 @@ + + 237 + @@ -305,6 +314,9 @@ + + 236 + @@ -319,7 +331,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src1 src1 @@ -331,7 +343,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src1 @@ -387,7 +399,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src1 Index: ql/src/test/results/compiler/plan/join7.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join7.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/join7.q.xml (working copy) @@ -140,6 +140,9 @@ + + 481 + @@ -207,6 +210,9 @@ + + 473 + @@ -479,6 +485,9 @@ + + 472 + @@ -518,6 +527,9 @@ + + 471 + @@ -652,6 +664,9 @@ + + 480 + @@ -719,6 +734,9 @@ + + 479 + @@ -979,6 +997,9 @@ + + 478 + @@ -1018,6 +1039,9 @@ + + 477 + @@ -1155,6 +1179,9 @@ + + 482 + @@ -1222,6 +1249,9 @@ + + 476 + @@ -1482,6 +1512,9 @@ + + 475 + @@ -1521,6 +1554,9 @@ + + 474 + @@ -1544,7 +1580,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src c:a:src1 @@ -1562,7 +1598,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -1618,7 +1654,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -1648,7 +1684,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/7114392/184401074.10001.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1828040/2977146.10001.insclause-0 @@ -1677,6 +1713,9 @@ + + 486 + @@ -1824,6 +1863,9 @@ + + 485 + @@ -1910,6 +1952,9 @@ + + 484 + @@ -2096,6 +2141,9 @@ + + 483 + Index: ql/src/test/results/compiler/plan/input7.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input7.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/input7.q.xml (working copy) @@ -31,7 +31,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/143269220.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1594657844.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -128,13 +128,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/143269220.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1594657844.10000.insclause-0 + + 249 + @@ -207,6 +210,9 @@ + + 248 + @@ -243,6 +249,9 @@ + + 247 + @@ -272,6 +281,9 @@ + + 244 + @@ -307,7 +319,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src1 src1 @@ -319,7 +331,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src1 @@ -375,7 +387,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src1 Index: ql/src/test/results/compiler/plan/input_testsequencefile.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input_testsequencefile.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/input_testsequencefile.q.xml (working copy) @@ -31,7 +31,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/376067338/60005233.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1276710261/568107444.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest4_sequencefile + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest4_sequencefile @@ -124,13 +124,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/376067338/60005233.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1276710261/568107444.10000.insclause-0 + + 287 + @@ -202,6 +205,9 @@ + + 286 + @@ -220,6 +226,9 @@ + + 285 + @@ -255,7 +264,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src src @@ -267,7 +276,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -323,7 +332,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/input8.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input8.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/input8.q.xml (working copy) @@ -26,7 +26,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/290201654.10001.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/158610613.10001.insclause-0 @@ -55,6 +55,9 @@ + + 259 + @@ -301,6 +304,9 @@ + + 258 + @@ -337,6 +343,9 @@ + + 257 + @@ -366,6 +375,9 @@ + + 254 + @@ -401,7 +413,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src1 src1 @@ -413,7 +425,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src1 @@ -469,7 +481,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src1 Index: ql/src/test/results/compiler/plan/join8.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join8.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/join8.q.xml (working copy) @@ -140,6 +140,9 @@ + + 510 + @@ -207,6 +210,9 @@ + + 505 + @@ -479,6 +485,9 @@ + + 504 + @@ -518,6 +527,9 @@ + + 503 + @@ -652,6 +664,9 @@ + + 509 + @@ -719,6 +734,9 @@ + + 508 + @@ -979,6 +997,9 @@ + + 507 + @@ -1018,6 +1039,9 @@ + + 506 + @@ -1041,7 +1065,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src c:a:src1 @@ -1056,7 +1080,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -1112,7 +1136,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -1146,7 +1170,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/18580553/574873966.10001.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/283039679/214129911.10001.insclause-0 @@ -1175,6 +1199,9 @@ + + 515 + @@ -1282,6 +1309,9 @@ + + 514 + @@ -1399,6 +1429,9 @@ + + 513 + @@ -1506,6 +1539,9 @@ + + 512 + @@ -1609,6 +1645,9 @@ + + 511 + Index: ql/src/test/results/compiler/plan/input9.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input9.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/input9.q.xml (working copy) @@ -31,7 +31,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/171676269.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1270426153.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -132,13 +132,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/171676269.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1270426153.10000.insclause-0 + + 271 + @@ -211,6 +214,9 @@ + + 270 + @@ -282,6 +288,9 @@ + + 269 + @@ -329,6 +338,9 @@ + + 268 + @@ -347,6 +359,9 @@ + + 264 + @@ -382,7 +397,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src1 src1 @@ -394,7 +409,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src1 @@ -450,7 +465,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src1 Index: ql/src/test/results/compiler/plan/union.q.xml =================================================================== --- ql/src/test/results/compiler/plan/union.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/union.q.xml (working copy) @@ -28,7 +28,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/83128511/25205679.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1016172561/566980907.10000.insclause-0 ../build/ql/test/data/warehouse/union.out @@ -79,7 +79,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/83128511/25205679.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1016172561/566980907.10000.insclause-0 @@ -108,6 +108,9 @@ + + 636 + @@ -182,6 +185,9 @@ + + 635 + @@ -200,6 +206,9 @@ + + 634 + @@ -241,6 +250,9 @@ + + 633 + @@ -363,6 +375,9 @@ + + 632 + @@ -374,6 +389,9 @@ + + 631 + @@ -509,6 +527,9 @@ + + 630 + @@ -633,6 +654,9 @@ + + 629 + @@ -672,6 +696,9 @@ + + 628 + @@ -690,7 +717,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src null-subquery1:unioninput-subquery1:src @@ -705,7 +732,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -761,7 +788,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/udf1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/udf1.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/udf1.q.xml (working copy) @@ -30,7 +30,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/35696865.10001.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/600185748.10001.insclause-0 @@ -59,6 +59,9 @@ + + 602 + @@ -1115,6 +1118,9 @@ + + 601 + @@ -1240,6 +1246,9 @@ + + 600 + @@ -1287,6 +1296,9 @@ + + 599 + @@ -1305,6 +1317,9 @@ + + 595 + @@ -1340,7 +1355,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src src @@ -1352,7 +1367,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -1408,7 +1423,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/udf4.q.xml =================================================================== --- ql/src/test/results/compiler/plan/udf4.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/udf4.q.xml (working copy) @@ -26,7 +26,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/151341829.10001.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/990628631.10001.insclause-0 @@ -55,6 +55,9 @@ + + 613 + @@ -1076,6 +1079,9 @@ + + 612 + @@ -1101,6 +1107,9 @@ + + 611 + @@ -1115,6 +1124,9 @@ + + 608 + @@ -1154,7 +1166,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 dest1 @@ -1166,7 +1178,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -1222,7 +1234,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 Index: ql/src/test/results/compiler/plan/input_testxpath.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input_testxpath.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/input_testxpath.q.xml (working copy) @@ -26,7 +26,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/405878710.10001.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/226690002.10001.insclause-0 @@ -55,6 +55,9 @@ + + 296 + @@ -229,6 +232,9 @@ + + 295 + @@ -285,6 +291,9 @@ + + 294 + @@ -334,6 +343,9 @@ + + 291 + @@ -413,7 +425,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src_thrift + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src_thrift src_thrift @@ -425,7 +437,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src_thrift + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src_thrift @@ -485,7 +497,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src_thrift + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src_thrift Index: ql/src/test/results/compiler/plan/udf6.q.xml =================================================================== --- ql/src/test/results/compiler/plan/udf6.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/udf6.q.xml (working copy) @@ -26,7 +26,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/602355685.10001.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1328628098.10001.insclause-0 @@ -55,6 +55,9 @@ + + 623 + @@ -221,6 +224,9 @@ + + 622 + @@ -246,6 +252,9 @@ + + 621 + @@ -260,6 +269,9 @@ + + 618 + @@ -295,7 +307,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src src @@ -307,7 +319,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -363,7 +375,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/input_part1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input_part1.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/input_part1.q.xml (working copy) @@ -26,7 +26,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/1714085878/13629975.10001.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1260276323/87357426.10001.insclause-0 @@ -55,6 +55,9 @@ + + 280 + @@ -166,6 +169,9 @@ + + 279 + @@ -443,6 +449,9 @@ + + 278 + @@ -502,6 +511,9 @@ + + 277 + @@ -516,7 +528,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcpart/hr=12/ds=2008-04-08 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcpart/hr=12/ds=2008-04-08 srcpart @@ -528,7 +540,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcpart/hr=12/ds=2008-04-08 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcpart/hr=12/ds=2008-04-08 @@ -597,7 +609,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcpart + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcpart Index: ql/src/test/results/compiler/plan/groupby1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby1.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/groupby1.q.xml (working copy) @@ -4,111 +4,9 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/3832508/94288076.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - columns.types - string:string - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - - - - - - - - - - - + - Stage-3 + Stage-1 @@ -118,257 +16,184 @@ - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/3832508/94288076.10001 - - - - - - - - - 0 - - - - - java.lang.String - - - - - - + + + + + + + + + + + + + true + + + /data/users/njain/hive1/hive/ql/../build/ql/tmp/169816496/1469104497.10000.insclause-0 + + + + + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + name + dest1 - - org.apache.hadoop.mapred.SequenceFileInputFormat + + columns.types + string:string - - org.apache.hadoop.mapred.SequenceFileOutputFormat + + serialization.ddl + struct dest1 { string key, string value} - - - - name - binary_sortable_table - - - serialization.sort.order - + - - - serialization.ddl - struct binary_sortable_table { string reducesinkkey0} - - - serialization.format - org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol - - + + serialization.format + 1 - - - - -1 - - - - - - -1 - - - - - - - 1 - - - - - java.lang.Double - - - - + + columns + key,value - - - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe + + bucket_count + -1 - - org.apache.hadoop.mapred.SequenceFileInputFormat + + serialization.lib + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - org.apache.hadoop.mapred.SequenceFileOutputFormat + + file.inputformat + org.apache.hadoop.mapred.TextInputFormat - - - - name - binary_table - - - serialization.ddl - struct binary_table { double reducesinkvalue0} - - - serialization.format - com.facebook.thrift.protocol.TBinaryProtocol - - + + file.outputformat + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + location + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 + + + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + - + + + + + + + + + + + + Stage-2 + + + + + + + src + + + + + + - - - - - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/3832508/94288076.10001 + + + + + + + + + 0 - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe + + + + java.lang.String - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_table - - - serialization.ddl - struct binary_table { string temporarycol0, double temporarycol1} - - - serialization.format - com.facebook.thrift.protocol.TBinaryProtocol - - - - - - - - - + + + + + + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - - - - - - 0 - - - - - - - - - - 1 - - - - - - - + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.mapred.SequenceFileOutputFormat + + + + + name + binary_sortable_table + + serialization.sort.order + + + + + serialization.ddl + struct binary_sortable_table { string reducesinkkey0} + + + serialization.format + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol + - - - - - - - - iterate - - + + -1 - + + + + + -1 + + - - - org.apache.hadoop.hive.ql.udf.UDAFSum + + + 1 - - - - - - org.apache.hadoop.hive.ql.udf.UDFToDouble - - - - evaluate - - - java.lang.String - - - - - - - - - - VALUE.0 - - - - - - - - - - - - + + + + java.lang.Double @@ -376,121 +201,117 @@ - - - - terminatePartial + + + + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.mapred.SequenceFileOutputFormat + + + + + name + binary_table + + + serialization.ddl + struct binary_table { double reducesinkvalue0} + + + serialization.format + com.facebook.thrift.protocol.TBinaryProtocol + + + - - + + + + 28 + + + + + + + + + + + + - - + + KEY.0 - + + + + + VALUE.0 + + + + + + - - - PARTIAL1 - - - - - - - - - - - - - - key - - - - - - - + + + + + + + + + + iterate + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDAFSum + + + + + + + org.apache.hadoop.hive.ql.udf.UDFToDouble - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_sortable_table - - - serialization.sort.order - + - - - serialization.ddl - struct binary_sortable_table { string reducesinkkey0} - - - serialization.format - org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol - - - + + + evaluate + + + java.lang.String + + - - -1 - - + - org.apache.hadoop.hive.ql.udf.UDFRand - - - - evaluate - - - - - - - - - - - - - - - -1 - - - - - - org.apache.hadoop.hive.ql.udf.UDFSubstr @@ -541,119 +362,55 @@ - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_table - - - serialization.ddl - struct binary_table { string reducesinkvalue0} - - - serialization.format - com.facebook.thrift.protocol.TBinaryProtocol - - - - + + - - - - - - - - - - - - - - - - - - - key - - - - - - - - - - value - - - - - - - - - - - - - - - - - - - - - - KEY.0 - - - - - - - - - - VALUE.0 - - - - - - - - - - - - + + + + + + terminatePartial + + + + + + key + + + + + + + + + + + HASH + + + + 27 + + + + + + + + @@ -661,7 +418,7 @@ - KEY.0 + 0 @@ -671,7 +428,7 @@ - VALUE.0 + 1 @@ -686,115 +443,181 @@ - - + + 26 - - -1 - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/3832508/94288076.10001 - + + + + - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/3832508/94288076.10001 + + + key + + + + + + + + + value + + + + + + - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/3832508/94288076.10001 - - - + + + + + + + + + -1 + + + + + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src + + + src + + + + + + + + + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src + + + + + + + + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + name + src + + columns.types + string:string + + + serialization.ddl + struct src { string key, string value} + + + serialization.format + 1 + + + columns + key,value + + + bucket_count + -1 + + + serialization.lib + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + + file.inputformat + org.apache.hadoop.mapred.TextInputFormat + + + file.outputformat + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + location + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src + + + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + - - + + + + + + + + + + - - + + + + + /data/users/njain/hive1/hive/ql/../build/ql/tmp/169816496/1469104497.10000.insclause-0 + + + + + + + + 31 + + - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/3832508/94288076.10000.insclause-0 - - - - - - - - - - - - - - - - - - - - - 0 - - - - - - - - - - 1 - - - - - - - - - - - + - - - - + + + + - - + + 0 - + - - + + 1 - + @@ -803,113 +626,83 @@ - - - - - - - - - - - - - - - - + + - merge - - - - - - - - - org.apache.hadoop.hive.ql.udf.UDAFSum + + + 0 - - - - - - VALUE.0 - - - - - - - + + - - - - - terminate - - - - - - - KEY.0 + 1 - + - - - FINAL - - + + 30 + - + - + + + + + + + + + + + + + + merge + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDAFSum + + + - - - 0 + + + VALUE.0 - - - - - - - - - 1 - - + @@ -920,129 +713,78 @@ - + - + terminate + + + + + + KEY.0 + + + + + + + + + + + FINAL + + - - - - - - Stage-2 - - - - - - - src - + + 29 - - - - - - - -1 - - - - - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + - src + - - - - - - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src - - - - - - - - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - src + + + + + + + + 0 - - columns.types - string:string + + - - serialization.ddl - struct src { string key, string value} - - - serialization.format + + + + + 1 - - columns - key,value + + - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src - - - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - - - + Index: ql/src/test/results/compiler/plan/groupby2.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby2.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/groupby2.q.xml (working copy) @@ -1,323 +1,125 @@ - - - - - - - Stage-3 - - - - - - - - - - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/27427164/760269758.10002 - - - - - - - - - 0 - - - - - java.lang.String - - - - - - - - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_sortable_table - - - serialization.sort.order - + - - - serialization.ddl - struct binary_sortable_table { string reducesinkkey0} - - - serialization.format - org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol - - - - - - - -1 - - - - - - -1 - - - - - - - 1 - - - - - java.lang.Double - - - - - - - - - 2 - - - - - java.lang.Long - - - - - - - - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_table - - - serialization.ddl - struct binary_table { double reducesinkvalue0, i64 reducesinkvalue1} - - - serialization.format - com.facebook.thrift.protocol.TBinaryProtocol - - - - - - - - + + + Stage-2 + + + + + + + src + + + + + + - - - - - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/27427164/760269758.10002 + + + + + + + + + 0 - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe + + + + java.lang.String - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_table - - - serialization.ddl - struct binary_table { string temporarycol0, double temporarycol1, i64 temporarycol2} - - - serialization.format - com.facebook.thrift.protocol.TBinaryProtocol - - - - - - - + + + + 1 + + + - - - - - - - - 0 - - - - - - - - - - 1 - - - - - - - - - - 2 - - - - - - - + + + + + + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.mapred.SequenceFileOutputFormat + + + + + name + binary_sortable_table + + serialization.sort.order + ++ + + + serialization.ddl + struct binary_sortable_table { string reducesinkkey0, string reducesinkkey1} + + + serialization.format + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol + - - - - - + + -1 + + - iterate + - - iterate - - + + -1 + + - - - org.apache.hadoop.hive.ql.udf.UDAFSum + + + 2 - - - - - - org.apache.hadoop.hive.ql.udf.UDFToDouble - - - - evaluate - - - java.lang.String - - - - - - - - - - KEY.1 - - - - - - - - - - - - + + + + java.lang.Double - - - org.apache.hadoop.hive.ql.udf.UDAFCount + + + 3 - - true - - - - - - - KEY.1 - - - - - + + + + java.lang.Long @@ -325,45 +127,137 @@ - - - - terminatePartial + + + + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - terminatePartial + + org.apache.hadoop.mapred.SequenceFileInputFormat + + org.apache.hadoop.mapred.SequenceFileOutputFormat + + + + + name + binary_table + + + serialization.ddl + struct binary_table { double reducesinkvalue0, i64 reducesinkvalue1} + + + serialization.format + com.facebook.thrift.protocol.TBinaryProtocol + + + - - + + + + 40 + + + + + + + + + + + + - - + + KEY.0 - + + + + + KEY.1 + + + + + + + + + + VALUE.0 + + + + + + + + + + VALUE.1 + + + + + + - - - PARTIAL1 - - - - - - - - - - + + + + + + + + + + iterate + + + iterate + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDAFSum + + + + + + + org.apache.hadoop.hive.ql.udf.UDFToDouble + + + + evaluate + + + java.lang.String + + + + + + @@ -372,16 +266,13 @@ evaluate - + java.lang.String java.lang.Integer - - java.lang.Integer - @@ -390,7 +281,7 @@ - key + value @@ -407,20 +298,10 @@ - 1 + 5 - - - - - - - 1 - - - @@ -428,213 +309,218 @@ + + + + + + + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDAFCount + + + true + + + + + + + org.apache.hadoop.hive.ql.udf.UDFSubstr + + + + evaluate + + + java.lang.String + + + java.lang.Integer + + + + + + - - - org.apache.hadoop.hive.ql.udf.UDFSubstr + + + value - - - evaluate - - - java.lang.String - - - java.lang.Integer - - - - - - - - - - value - - - - - - - - - - - - - 5 - - - - - - - - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_sortable_table + + + + - - serialization.sort.order - ++ + + 5 - - serialization.ddl - struct binary_sortable_table { string reducesinkkey0, string reducesinkkey1} - - - serialization.format - org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol - - - -1 + + - - + + + + + + + + + + + + terminatePartial + + + terminatePartial + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDFSubstr + + + + evaluate + + + java.lang.String + + + java.lang.Integer + + + java.lang.Integer + + + + + + + + + + key - - -1 + + - - + + + + + + - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_table - - - serialization.ddl - struct binary_table { } - - - serialization.format - com.facebook.thrift.protocol.TBinaryProtocol - - - - + + 1 - - - - - - - - - - - - - - - - - - - key - - - - - - - - - - value - - - - - - - - - - - + + + + + + 1 + - - - - - - - - KEY.0 - - - - - - - - - - KEY.1 - - - - - - - + + + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDFSubstr + + + + evaluate + + + java.lang.String + + + java.lang.Integer + + + + + + + + + + value + + + + + + + + + + 5 + + + + + + - - - + + + HASH + + + + 39 + + + + + + + + @@ -642,7 +528,7 @@ - KEY.0 + 0 @@ -652,9 +538,19 @@ - VALUE.0 + 1 + + + + + + + + 2 + + @@ -662,7 +558,7 @@ - VALUE.1 + 3 @@ -677,218 +573,213 @@ - - + + 38 - - -1 - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/27427164/760269758.10002 - + + + + - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/27427164/760269758.10002 + + + key + + + + + + + + + value + + + + + + - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/27427164/760269758.10002 - - - + + + + + + + + + -1 + + + + + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src + + + src + + + + + + + + + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src + + + + + + + + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + name + src + + columns.types + string:string + + + serialization.ddl + struct src { string key, string value} + + + serialization.format + 1 + + + columns + key,value + + + bucket_count + -1 + + + serialization.lib + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + + file.inputformat + org.apache.hadoop.mapred.TextInputFormat + + + file.outputformat + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + location + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src + + + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + - - + + + + + + + + + + - - - - - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/27427164/760269758.10001.insclause-0 - - - - - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - columns - _c0,_c1,_c2 - - - serialization.format - 1 - - - - - - + + + + + /data/users/njain/hive1/hive/ql/../build/ql/tmp/717653767/1509670805.10001.insclause-0 + + + + + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - - - - + + org.apache.hadoop.mapred.TextInputFormat - - - - - - - - 0 - - - - - - - - - - 1 - - - - - - - - - - 2 - - - - - - - + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + _c0,_c1,_c2 + + serialization.format + 1 + - - - - + + 43 + + + + + + + + + + + + - - + + 0 - + - - - 2 + + + 1 - + - - - org.apache.hadoop.hive.ql.udf.UDFConcat + + + 2 - - - evaluate - - - java.lang.String - - - java.lang.String - - - - - - - - - - 0 - - - - - - - - - - org.apache.hadoop.hive.ql.udf.UDFToString - - - - evaluate - - - java.lang.Double - - - - - - - - - - 1 - - - - - - - - - - - - - - - - + @@ -897,99 +788,100 @@ - - - - - - - - - - - - - - - - + + - merge + + + 0 + + + + + - merge + + + 2 + + + + + - - - - - - - org.apache.hadoop.hive.ql.udf.UDAFSum + + + org.apache.hadoop.hive.ql.udf.UDFConcat - + + + evaluate + + + java.lang.String + + + java.lang.String + + + + + - VALUE.0 + 0 - + - - - - - - - - org.apache.hadoop.hive.ql.udf.UDAFCount - - - - - - VALUE.1 + + + org.apache.hadoop.hive.ql.udf.UDFToString + + + evaluate + + + java.lang.Double + + + + + + + + + + 1 + + + + + + + + - + - - - - - - - - terminate - - - terminate - - - - - - - - - KEY.0 - @@ -997,50 +889,77 @@ - - - FINAL - - + + 42 + - + - + + + + + + + + + + + + + + merge + + + merge + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDAFSum + + + - - - 0 + + + VALUE.0 - - - - - - - - - 1 - - + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDAFCount + + + - - - 2 + + + VALUE.1 - + @@ -1051,129 +970,91 @@ - + - + terminate + + terminate + + + + + + + KEY.0 + + + + + + + + + + + FINAL + + - - - - - - Stage-2 - - - - - - - src - + + 41 - - - - - - - -1 - - - - - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + - src + - - - - - - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src - - - - - - - - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - src + + + + + + + + 0 - - columns.types - string:string + + - - serialization.ddl - struct src { string key, string value} - - - serialization.format + + + + + 1 - - columns - key,value + + - - bucket_count - -1 + + + + + + 2 - - serialization.lib - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src - - - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - - - + Index: ql/src/test/results/compiler/plan/groupby3.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby3.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/groupby3.q.xml (working copy) @@ -1,182 +1,50 @@ - - - - - - - Stage-3 - - - - - - - - - - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/237748051/1295963032.10002 - - - - - - - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_sortable_table - - - serialization.sort.order - - - - serialization.ddl - struct binary_sortable_table { } - - - serialization.format - org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol - - - - - - - -1 - - - - - - -1 - - - - - - - 0 - - - - - java.lang.Double - - - - - - - - - 1 - - - - - java.lang.String - - - - - - - - - 2 - - - - - - - - - - 3 - - - - - - - - - - 4 - - - - - - - - - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_table - - - serialization.ddl - struct binary_table { double reducesinkvalue0, string reducesinkvalue1, string reducesinkvalue2, string reducesinkvalue3, string reducesinkvalue4} - - - serialization.format - com.facebook.thrift.protocol.TBinaryProtocol - - - - - - - - + + + Stage-2 + + + + + + + src + + + + + + - + - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/237748051/1295963032.10002 + + + + + + + 0 + + + + + java.lang.String + + + + + + - - + + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe @@ -190,11 +58,111 @@ name + binary_sortable_table + + + serialization.sort.order + + + + + serialization.ddl + struct binary_sortable_table { string reducesinkkey0} + + + serialization.format + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol + + + + + + + -1 + + + + + + -1 + + + + + + + 1 + + + + + java.lang.Double + + + + + + + + + 2 + + + + + + + + + + 3 + + + + + + + + + + 4 + + + + + + + + + + 5 + + + + + + + + + + + + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.mapred.SequenceFileOutputFormat + + + + + name binary_table serialization.ddl - struct binary_table { double temporarycol0, string temporarycol1, string temporarycol2, string temporarycol3, string temporarycol4} + struct binary_table { double reducesinkvalue0, string reducesinkvalue1, string reducesinkvalue2, string reducesinkvalue3, string reducesinkvalue4} serialization.format @@ -206,6 +174,9 @@ + + 58 + @@ -214,13 +185,13 @@ - + - 0 + KEY.0 @@ -230,7 +201,7 @@ - 1 + VALUE.0 @@ -240,33 +211,43 @@ - 2 + VALUE.1 - + - 3 + VALUE.2 - + - 4 + VALUE.3 - + + + + + VALUE.4 + + + + + + @@ -323,19 +304,60 @@ - - - KEY.0 + + + org.apache.hadoop.hive.ql.udf.UDFSubstr + + + evaluate + + + java.lang.String + + + java.lang.Integer + + + + + + + + + + 0 + + + + + + + + + + + + java.lang.Integer + + + + + 5 + + + + + - + - + @@ -351,41 +373,20 @@ - - - KEY.0 - - - - - - - - - - - - - - org.apache.hadoop.hive.ql.udf.UDAFAvg - - - true - - - - - org.apache.hadoop.hive.ql.udf.UDFToDouble + org.apache.hadoop.hive.ql.udf.UDFSubstr - + evaluate - + java.lang.String + + java.lang.Integer + @@ -394,13 +395,23 @@ - KEY.0 + 0 - + + + + + + + + 5 + + + @@ -415,29 +426,11 @@ - org.apache.hadoop.hive.ql.udf.UDAFMin$MinStringEvaluator + org.apache.hadoop.hive.ql.udf.UDAFAvg - - - - - - KEY.0 - - - - - - - + + true - - - - - - org.apache.hadoop.hive.ql.udf.UDAFAvg - @@ -458,19 +451,56 @@ - - - KEY.0 + + + org.apache.hadoop.hive.ql.udf.UDFSubstr + + + evaluate + + + java.lang.String + + + java.lang.Integer + + + + + + + + + + 0 + + + + + + + + + + + + + 5 + + + + + - + - + @@ -478,45 +508,13 @@ - - - - - terminatePartial - - - terminatePartial - - - terminatePartial - - - terminatePartial - - - terminatePartial - - - - - - - - - PARTIAL1 - - - - - - - - - - - - + + + org.apache.hadoop.hive.ql.udf.UDAFMin$MinStringEvaluator + + + @@ -543,18 +541,14 @@ 0 - + - - - java.lang.Integer - - + 5 @@ -564,265 +558,299 @@ - + - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_sortable_table - - - serialization.sort.order - + - - - serialization.ddl - struct binary_sortable_table { string reducesinkkey0} - - - serialization.format - org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol - - - - + + + + + + org.apache.hadoop.hive.ql.udf.UDAFAvg - - -1 - - - - - - -1 - - - - - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_table + + + + + + org.apache.hadoop.hive.ql.udf.UDFToDouble - - serialization.ddl - struct binary_table { } + + + evaluate + + + java.lang.String + + + - - serialization.format - com.facebook.thrift.protocol.TBinaryProtocol - - - - - - - - - - - - - - - - - - - - - + - - - value + + + org.apache.hadoop.hive.ql.udf.UDFSubstr - - + + + evaluate + + + java.lang.String + + + java.lang.Integer + + + - - - - - - - - - - - - - - - - - - - - - + + - - - key + + + 0 - - + + - - - value + + + - - + + 5 - - - - - - - - - - - - - - 0 + + - - - + + + - - - - + + + + + + terminatePartial + + + terminatePartial + + + terminatePartial + + + terminatePartial + + + terminatePartial + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDFSubstr + + + + evaluate + + + java.lang.String + + + java.lang.Integer + + + + + + - - - KEY.0 + + + 0 - - + + + + + + + + + 5 + + + + + + + + + HASH + + + + 57 + + + + + + + + - + + + + + + + 0 + + + + + + + + + + 1 + + + + + + + + + + 2 + + + + + + + + + + 3 + + + + + + + + + + 4 + + + + + + + + + + 5 + + + + + + + + + - - - - + + + + - - - VALUE.0 + + + value - + + + + + + + 56 + + + + + + + + + + + + - VALUE.1 + 0 - + - - - - VALUE.2 - - - - - - - - - - VALUE.3 - - - - - - - - - - VALUE.4 - - - - - - @@ -831,198 +859,234 @@ - - + + 50 - - -1 - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/237748051/1295963032.10002 - + + + + - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/237748051/1295963032.10002 + + + key + + + + + + + + + value + + + + + + - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/237748051/1295963032.10002 - - - + + + + + + + + + -1 + + + + + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src + + + src + + + + + + + + + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src + + + + + + + + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + name + src + + columns.types + string:string + + + serialization.ddl + struct src { string key, string value} + + + serialization.format + 1 + + + columns + key,value + + + bucket_count + -1 + + + serialization.lib + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + + file.inputformat + org.apache.hadoop.mapred.TextInputFormat + + + file.outputformat + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + location + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src + + + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + - - + + + + + + + + + + - - - - - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/832745015.10001.insclause-0 - - - - - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - columns - _c0,_c1,_c2,_c3,_c4 - - - serialization.format - 1 - - - - - - + + + + + /data/users/njain/hive1/hive/ql/../build/ql/tmp/334497369.10001.insclause-0 + + + + + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - - - - + + org.apache.hadoop.mapred.TextInputFormat - - - - - - - - 0 - - - - - - - - - - 1 - - - - - - - - - - 2 - - - - - - - - - - 3 - - - - - - - - - - 4 - - - - - - - + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + _c0,_c1,_c2,_c3,_c4 + + serialization.format + 1 + - - - - + + 61 + + + + + + + + + + + + - - + + 0 - - + + - - - 4 + + + 1 - - + + - - + + 2 - - + + - - - 1 + + + 3 - - + + - - - 3 + + + 4 - - + + @@ -1030,240 +1094,209 @@ - - - - - - - - - - - - - - - - + + - merge - - - merge - - - merge - - - merge - - - merge - - - - - - - - - org.apache.hadoop.hive.ql.udf.UDAFSum + + + 0 - - - - - - VALUE.0 - - - - - - - + + - - - org.apache.hadoop.hive.ql.udf.UDAFMax$MaxStringEvaluator + + + 4 - - - - - - VALUE.1 - - - - - - - + + - - - org.apache.hadoop.hive.ql.udf.UDAFAvg + + + 2 - - - - - - VALUE.2 - - - - - - - + + - - - org.apache.hadoop.hive.ql.udf.UDAFMin$MinStringEvaluator + + + 1 - - - - - - VALUE.3 - - - - - - - + + - - - org.apache.hadoop.hive.ql.udf.UDAFAvg + + + 3 - - - - - - VALUE.4 - - - - - - - + + - - - - terminate - - - terminate - - - terminate - - - terminate - - - terminate - - - - - - - - - FINAL - - + + 60 + - + - + + + + + + + + + + + + + + merge + + + merge + + + merge + + + merge + + + merge + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDAFSum + + + - - - 0 + + + VALUE.0 - - + + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDAFMax$MaxStringEvaluator + + + - - - 1 + + + VALUE.1 - - + + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDAFAvg + + + - - - 2 + + + VALUE.2 - + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDAFMin$MinStringEvaluator + + + - - - 3 + + + VALUE.3 - - + + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDAFAvg + + + - - - 4 + + + VALUE.4 - + @@ -1274,129 +1307,109 @@ - + - + terminate + + terminate + + + terminate + + + terminate + + + terminate + + + + + + + FINAL + + - - - - - - Stage-2 - - - - - - - src - + + 59 - - - - - - - -1 - - - - - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + - src + - - - - - - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src - - - - - - - - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - src + + + + + + + + 0 - - columns.types - string:string + + - - serialization.ddl - struct src { string key, string value} - - - serialization.format + + + + + 1 - - columns - key,value + + - - bucket_count - -1 + + + + + + 2 - - serialization.lib - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat + + + + + + 3 - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + - - location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + + + + + + 4 + + + - - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - - - + Index: ql/src/test/results/compiler/plan/subq.q.xml =================================================================== --- ql/src/test/results/compiler/plan/subq.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/subq.q.xml (working copy) @@ -28,7 +28,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/46472807/242639424.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/426119477/1401965151.10000.insclause-0 ../build/ql/test/data/warehouse/union.out @@ -75,7 +75,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/46472807/242639424.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/426119477/1401965151.10000.insclause-0 @@ -104,6 +104,9 @@ + + 589 + @@ -178,6 +181,9 @@ + + 588 + @@ -227,6 +233,9 @@ + + 587 + @@ -377,6 +386,9 @@ + + 586 + @@ -416,6 +428,9 @@ + + 585 + @@ -430,7 +445,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src unioninput:src @@ -442,7 +457,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -498,7 +513,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/groupby4.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby4.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/groupby4.q.xml (working copy) @@ -1,304 +1,50 @@ - - - - - - - Stage-3 - - - - - - - - - - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/27342663/186937414.10002 - - - - - - - - - 0 - - - - - java.lang.String - - - - - - - - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_sortable_table - - - serialization.sort.order - + - - - serialization.ddl - struct binary_sortable_table { string reducesinkkey0} - - - serialization.format - org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol - - - - - - - -1 - - - - - - -1 - - - - - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_table - - - serialization.ddl - struct binary_table { } - - - serialization.format - com.facebook.thrift.protocol.TBinaryProtocol - - - - - - - - + + + Stage-2 + + + + + + + src + + + + + + - + - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/27342663/186937414.10002 - - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_table - - - serialization.ddl - struct binary_table { string temporarycol0} - - - serialization.format - com.facebook.thrift.protocol.TBinaryProtocol - - - - - - - - - - - - - - - - - - - - - - 0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - KEY.0 - - - - - - - - - - - PARTIAL1 - - - - - - - - - - + - - - org.apache.hadoop.hive.ql.udf.UDFSubstr + + + 0 - - - evaluate - - - java.lang.String - - - java.lang.Integer - - - java.lang.Integer - - - - - - - - - - 0 - - - - - + + + + java.lang.String - - - - - - java.lang.Integer - - - - - 1 - - - - - - - - - - 1 - - - - - - - + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe @@ -334,31 +80,7 @@ -1 - - - - - org.apache.hadoop.hive.ql.udf.UDFRand - - - - evaluate - - - - - - - - - - java.lang.Double - - - - - - + -1 @@ -367,7 +89,7 @@ - + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe @@ -397,97 +119,13 @@ + + 77 + - - - - - - - - - - - - - - - - key - - - - - - - - - - - - - - - - - - - - - - - - - - - - - key - - - - - - - - - - value - - - - - - - - - - - - - - - - - - - - - - 0 - - - - - - - - - - - + @@ -513,259 +151,211 @@ - - - - - - - - - - - - - - - KEY.0 + + + + - - + + - - - - - - - - - - - - - - - -1 - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/27342663/186937414.10002 - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/27342663/186937414.10002 - - - - - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/27342663/186937414.10002 - - - - - - - - - - - - - - - - - - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/1393258827.10001.insclause-0 - - - - - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDFSubstr - - org.apache.hadoop.mapred.TextInputFormat + + + evaluate + + + java.lang.String + + + java.lang.Integer + + + java.lang.Integer + + + - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - columns - _c0 + + + + + + 0 + + + + + - - serialization.format - 1 + + + + + + java.lang.Integer + + + + + 1 + + + + + + + + + 1 + + + + + + - - - - - + + + HASH - - - - - - - - 0 - - - - - - - - - - - - - - - + + 76 + + - - - 0 + + + + + + + + + + + + 0 + + + + + - - - - + + + + + - + + + key + + + + + - - - - - - - - - - - - - + + 75 - - - - - - - + - - - KEY.0 + + + + + + + + + + + + 0 + + + + + - - - - - - FINAL - - - - + + + + 69 + + + + + - + + + key + + + + + - - - - - - - - - - 0 - - - - - + + + + value + + + - - - - - - - - - - - Stage-2 - - - - - - - src - - - - - + -1 @@ -773,7 +363,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src src @@ -785,7 +375,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -841,7 +431,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -855,12 +445,184 @@ - + + + + + + + + + + + + + /data/users/njain/hive1/hive/ql/../build/ql/tmp/69474526.10001.insclause-0 + + + + + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + _c0 + + + serialization.format + 1 + + + + + + + + + 80 + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + 79 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + KEY.0 + + + + + + + + + + + FINAL + + + + + + 78 + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + - + Index: ql/src/test/results/compiler/plan/groupby5.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby5.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/groupby5.q.xml (working copy) @@ -1,272 +1,97 @@ - - - - - - - Stage-3 - - - - - - - - - - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/923313397/1330191211.10002 - - - - - - - - - 0 - - - - - java.lang.String - - - - - - - - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_sortable_table - - - serialization.sort.order - + - - - serialization.ddl - struct binary_sortable_table { string reducesinkkey0} - - - serialization.format - org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol - - - - - - - -1 - - - - - - -1 - - - - - - - 1 - - - - - java.lang.Double - - - - - - - - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_table - - - serialization.ddl - struct binary_table { double reducesinkvalue0} - - - serialization.format - com.facebook.thrift.protocol.TBinaryProtocol - - - - - - - - + + + Stage-2 + + + + + + + src + + + + + + - - - - - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/923313397/1330191211.10002 + + + + + + + + + 0 - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe + + + + java.lang.String - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_table - - - serialization.ddl - struct binary_table { string temporarycol0, double temporarycol1} - - - serialization.format - com.facebook.thrift.protocol.TBinaryProtocol - - - - - - - - - + + + + + + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - - - - - - 0 - - - - - - - - - - 1 - - - - - - - + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.mapred.SequenceFileOutputFormat + + + + + name + binary_sortable_table + + serialization.sort.order + + + + + serialization.ddl + struct binary_sortable_table { string reducesinkkey0} + + + serialization.format + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol + - - - - - - - - iterate - - + + -1 - + + + + + -1 + + - - - org.apache.hadoop.hive.ql.udf.UDAFSum + + + 1 - - - - - - org.apache.hadoop.hive.ql.udf.UDFToDouble - - - - evaluate - - - java.lang.String - - - - - - - - - - VALUE.0 - - - - - - - - - - - - + + + + java.lang.Double @@ -274,121 +99,117 @@ - - - - terminatePartial + + + + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.mapred.SequenceFileOutputFormat + + + + + name + binary_table + + + serialization.ddl + struct binary_table { double reducesinkvalue0} + + + serialization.format + com.facebook.thrift.protocol.TBinaryProtocol + + + - - + + + + 90 + + + + + + + + + + + + - - + + KEY.0 - + + + + + VALUE.0 + + + + + + - - - PARTIAL1 - - - - - - - - - - - - - - key - - - - - - - + + + + + + + + + + iterate + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDAFSum + + + + + + + org.apache.hadoop.hive.ql.udf.UDFToDouble - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_sortable_table - - - serialization.sort.order - + - - - serialization.ddl - struct binary_sortable_table { string reducesinkkey0} - - - serialization.format - org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol - - - + + + evaluate + + + java.lang.String + + - - -1 - - + - org.apache.hadoop.hive.ql.udf.UDFRand - - - - evaluate - - - - - - - - - - - - - - - -1 - - - - - - org.apache.hadoop.hive.ql.udf.UDFSubstr @@ -439,119 +260,55 @@ - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_table - - - serialization.ddl - struct binary_table { string reducesinkvalue0} - - - serialization.format - com.facebook.thrift.protocol.TBinaryProtocol - - - - + + - - - - - - - - - - - - - - - - - - - key - - - - - - - - - - value - - - - - - - - - - - - - - - - - - - - - - KEY.0 - - - - - - - - - - VALUE.0 - - - - - - - - - - - - + + + + + + terminatePartial + + + + + + key + + + + + + + + + + + HASH + + + + 89 + + + + + + + + @@ -559,7 +316,7 @@ - KEY.0 + 0 @@ -569,7 +326,7 @@ - VALUE.0 + 1 @@ -584,137 +341,203 @@ - - + + 88 - - -1 - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/923313397/1330191211.10002 - + + + + - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/923313397/1330191211.10002 + + + key + + + + + + + + + value + + + + + + - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/923313397/1330191211.10002 - - - + + + + + + + + + -1 + + + + + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src + + + src + + + + + + + + + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src + + + + + + + + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + name + src + + columns.types + string:string + + + serialization.ddl + struct src { string key, string value} + + + serialization.format + 1 + + + columns + key,value + + + bucket_count + -1 + + + serialization.lib + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + + file.inputformat + org.apache.hadoop.mapred.TextInputFormat + + + file.outputformat + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + location + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src + + + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + - - + + + + + + + + + + - - - - - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/923313397/1330191211.10001.insclause-0 - - - - - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - columns - src.key,_c1 - - - serialization.format - 1 - - - - - - + + + + + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1489470041/391886937.10001.insclause-0 + + + + + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - - - - + + org.apache.hadoop.mapred.TextInputFormat - - - - - - - - 0 - - - - - - - - - - 1 - - - - - - - + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + src.key,_c1 + + serialization.format + 1 + - - - - + + 93 + + + + + + + + + + + + - - + + 0 - + - - + + 1 - + @@ -723,113 +546,83 @@ - - - - - - - - - - - - - - - - + + - merge - - - - - - - - - org.apache.hadoop.hive.ql.udf.UDAFSum + + + 0 - - - - - - VALUE.0 - - - - - - - + + - - - - - terminate - - - - - - - KEY.0 + 1 - + - - - FINAL - - + + 92 + - + - + + + + + + + + + + + + + + merge + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDAFSum + + + - - - 0 + + + VALUE.0 - - - - - - - - - 1 - - + @@ -840,129 +633,78 @@ - + - + terminate + + + + + + KEY.0 + + + + + + + + + + + FINAL + + - - - - - - Stage-2 - - - - - - - src - + + 91 - - - - - - - -1 - - - - - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + - src + - - - - - - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src - - - - - - - - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - src + + + + + + + + 0 - - columns.types - string:string + + - - serialization.ddl - struct src { string key, string value} - - - serialization.format + + + + + 1 - - columns - key,value + + - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src - - - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - - - - + Index: ql/src/test/results/compiler/plan/groupby6.q.xml =================================================================== --- ql/src/test/results/compiler/plan/groupby6.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/groupby6.q.xml (working copy) @@ -1,304 +1,50 @@ - - - - - - - Stage-3 - - - - - - - - - - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/34538901/695004770.10002 - - - - - - - - - 0 - - - - - java.lang.String - - - - - - - - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_sortable_table - - - serialization.sort.order - + - - - serialization.ddl - struct binary_sortable_table { string reducesinkkey0} - - - serialization.format - org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol - - - - - - - -1 - - - - - - -1 - - - - - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_table - - - serialization.ddl - struct binary_table { } - - - serialization.format - com.facebook.thrift.protocol.TBinaryProtocol - - - - - - - - + + + Stage-2 + + + + + + + src + + + + + + - + - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/34538901/695004770.10002 - - - - - org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.mapred.SequenceFileOutputFormat - - - - - name - binary_table - - - serialization.ddl - struct binary_table { string temporarycol0} - - - serialization.format - com.facebook.thrift.protocol.TBinaryProtocol - - - - - - - - - - - - - - - - - - - - - - 0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - KEY.0 - - - - - - - - - - - PARTIAL1 - - - - - - - - - - + - - - org.apache.hadoop.hive.ql.udf.UDFSubstr + + + 0 - - - evaluate - - - java.lang.String - - - java.lang.Integer - - - java.lang.Integer - - - - - - - - - - 0 - - - - - + + + + java.lang.String - - - - - - java.lang.Integer - - - - - 5 - - - - - - - - - - 1 - - - - - - - + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe @@ -334,31 +80,7 @@ -1 - - - - - org.apache.hadoop.hive.ql.udf.UDFRand - - - - evaluate - - - - - - - - - - java.lang.Double - - - - - - + -1 @@ -367,7 +89,7 @@ - + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe @@ -397,97 +119,13 @@ + + 108 + - - - - - - - - - - - - - - - - value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - key - - - - - - - - - - value - - - - - - - - - - - - - - - - - - - - - - 0 - - - - - - - - - - - + @@ -513,259 +151,211 @@ - - - - - - - - - - - - - - - KEY.0 + + + + - - + + - - - - - - - - - - - - - - - -1 - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/34538901/695004770.10002 - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/34538901/695004770.10002 - - - - - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/tmp/34538901/695004770.10002 - - - - - - - - - - - - - - - - - - - - - - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/1392764200.10001.insclause-0 - - - - - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDFSubstr - - org.apache.hadoop.mapred.TextInputFormat + + + evaluate + + + java.lang.String + + + java.lang.Integer + + + java.lang.Integer + + + - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - columns - _c0 + + + + + + 0 + + + + + - - serialization.format - 1 + + + + + + java.lang.Integer + + + + + 5 + + + + + + + + + 1 + + + + + + - - - - - + + + HASH - - - - - - - - 0 - - - - - - - - - - - - - - - + + 107 + + - - - 0 + + + + + + + + + + + + 0 + + + + + - - - - + + + + + - + + + value + + + + + - - - - - - - - - - - - - + + 106 - - - - - - - + - - - KEY.0 + + + + + + + + + + + + 0 + + + + + - - - - - - FINAL - - - - + + + + 100 + + + + + - + + + key + + + + + - - - - - - - - - - 0 - - - - - + + + + value + + + - - - - - - - - - - - Stage-2 - - - - - - - src - - - - - + -1 @@ -773,7 +363,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src src @@ -785,7 +375,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -841,7 +431,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -855,12 +445,184 @@ - + + + + + + + + + + + + + /data/users/njain/hive1/hive/ql/../build/ql/tmp/889541974.10001.insclause-0 + + + + + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + _c0 + + + serialization.format + 1 + + + + + + + + + 111 + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + 110 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + KEY.0 + + + + + + + + + + + FINAL + + + + + + 109 + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + - + Index: ql/src/test/results/compiler/plan/case_sensitivity.q.xml =================================================================== --- ql/src/test/results/compiler/plan/case_sensitivity.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/case_sensitivity.q.xml (working copy) @@ -31,7 +31,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/1261061735.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/152837755.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -132,13 +132,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/1261061735.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/152837755.10000.insclause-0 + + 7 + @@ -269,6 +272,9 @@ + + 6 + @@ -359,6 +365,9 @@ + + 5 + @@ -426,6 +435,9 @@ + + 4 + @@ -444,6 +456,9 @@ + + 0 + @@ -530,7 +545,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src_thrift + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src_thrift src_thrift @@ -542,7 +557,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src_thrift + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src_thrift @@ -602,7 +617,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src_thrift + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src_thrift Index: ql/src/test/results/compiler/plan/input20.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input20.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/input20.q.xml (working copy) @@ -155,6 +155,9 @@ + + 158 + @@ -255,6 +258,9 @@ + + 157 + @@ -479,6 +485,9 @@ + + 156 + @@ -536,6 +545,9 @@ + + 155 + @@ -565,6 +577,9 @@ + + 147 + @@ -606,7 +621,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src tmap:src @@ -618,7 +633,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -674,7 +689,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -704,7 +719,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/16892348.10001.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1576892419.10001.insclause-0 @@ -733,6 +748,9 @@ + + 162 + @@ -829,6 +847,9 @@ + + 161 + @@ -875,6 +896,9 @@ + + 160 + @@ -928,6 +952,9 @@ + + 159 + Index: ql/src/test/results/compiler/plan/sample1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample1.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/sample1.q.xml (working copy) @@ -30,7 +30,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/169301459/102935396.10001.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/128865599/7453289.10001.insclause-0 @@ -59,6 +59,9 @@ + + 533 + @@ -173,6 +176,9 @@ + + 532 + @@ -320,6 +326,9 @@ + + 531 + @@ -545,6 +554,9 @@ + + 530 + @@ -556,6 +568,9 @@ + + 529 + @@ -570,7 +585,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcpart/hr=11/ds=2008-04-08 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcpart/hr=11/ds=2008-04-08 s @@ -582,7 +597,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcpart/hr=11/ds=2008-04-08 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcpart/hr=11/ds=2008-04-08 @@ -651,7 +666,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcpart + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcpart Index: ql/src/test/results/compiler/plan/sample2.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample2.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/sample2.q.xml (working copy) @@ -31,7 +31,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/158143647/19880045.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/307354094/106277115.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -124,13 +124,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/158143647/19880045.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/307354094/106277115.10000.insclause-0 + + 541 + @@ -205,6 +208,9 @@ + + 540 + @@ -223,6 +229,9 @@ + + 539 + @@ -258,7 +267,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket/kv1.txt + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket/kv1.txt s @@ -270,7 +279,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket/kv1.txt + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket/kv1.txt @@ -330,7 +339,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket Index: ql/src/test/results/compiler/plan/sample3.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample3.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/sample3.q.xml (working copy) @@ -31,7 +31,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/362417800/1298736006.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/337592833/305294881.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -128,13 +128,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/362417800/1298736006.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/337592833/305294881.10000.insclause-0 + + 548 + @@ -209,6 +212,9 @@ + + 547 + @@ -397,6 +403,9 @@ + + 546 + @@ -408,6 +417,9 @@ + + 545 + @@ -443,7 +455,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket s @@ -455,7 +467,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket @@ -515,7 +527,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket Index: ql/src/test/results/compiler/plan/sample4.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample4.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/sample4.q.xml (working copy) @@ -31,7 +31,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/637806170/200924363.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/53057454/118359877.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -124,13 +124,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/637806170/200924363.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/53057454/118359877.10000.insclause-0 + + 555 + @@ -205,6 +208,9 @@ + + 554 + @@ -223,6 +229,9 @@ + + 553 + @@ -258,7 +267,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket/kv1.txt + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket/kv1.txt s @@ -270,7 +279,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket/kv1.txt + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket/kv1.txt @@ -330,7 +339,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket Index: ql/src/test/results/compiler/plan/sample5.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample5.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/sample5.q.xml (working copy) @@ -31,7 +31,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/1281636920/855112172.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/491181944/137858642.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -128,13 +128,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/1281636920/855112172.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/491181944/137858642.10000.insclause-0 + + 562 + @@ -209,6 +212,9 @@ + + 561 + @@ -384,6 +390,9 @@ + + 560 + @@ -395,6 +404,9 @@ + + 559 + @@ -430,7 +442,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket s @@ -442,7 +454,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket @@ -502,7 +514,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket Index: ql/src/test/results/compiler/plan/sample6.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample6.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/sample6.q.xml (working copy) @@ -31,7 +31,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/178457491/743576201.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1205872150/8563070.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -128,13 +128,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/178457491/743576201.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1205872150/8563070.10000.insclause-0 + + 570 + @@ -209,6 +212,9 @@ + + 569 + @@ -384,6 +390,9 @@ + + 568 + @@ -395,6 +404,9 @@ + + 567 + @@ -430,7 +442,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket/kv1.txt + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket/kv1.txt s @@ -442,7 +454,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket/kv1.txt + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket/kv1.txt @@ -502,7 +514,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket Index: ql/src/test/results/compiler/plan/sample7.q.xml =================================================================== --- ql/src/test/results/compiler/plan/sample7.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/sample7.q.xml (working copy) @@ -31,7 +31,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/353480016/1067577190.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/324580728/93527081.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -132,13 +132,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/353480016/1067577190.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/324580728/93527081.10000.insclause-0 + + 579 + @@ -213,6 +216,9 @@ + + 578 + @@ -342,6 +348,9 @@ + + 577 + @@ -530,6 +539,9 @@ + + 576 + @@ -541,6 +553,9 @@ + + 575 + @@ -555,7 +570,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket/kv1.txt + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket/kv1.txt s @@ -567,7 +582,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket/kv1.txt + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket/kv1.txt @@ -627,7 +642,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/srcbucket + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/srcbucket Index: ql/src/test/results/compiler/plan/cast1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/cast1.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/cast1.q.xml (working copy) @@ -30,7 +30,7 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/394814637.10001.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/86110692.10001.insclause-0 @@ -59,6 +59,9 @@ + + 20 + @@ -543,6 +546,9 @@ + + 19 + @@ -664,6 +670,9 @@ + + 18 + @@ -711,6 +720,9 @@ + + 17 + @@ -729,6 +741,9 @@ + + 13 + @@ -764,7 +779,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src src @@ -776,7 +791,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -832,7 +847,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src Index: ql/src/test/results/compiler/plan/join1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/join1.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/join1.q.xml (working copy) @@ -31,7 +31,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/129851762.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/283280759.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -234,6 +234,9 @@ + + 323 + @@ -273,6 +276,9 @@ + + 314 + @@ -414,6 +420,9 @@ + + 322 + @@ -461,6 +470,9 @@ + + 321 + @@ -490,6 +502,9 @@ + + 315 + @@ -534,7 +549,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src src2 @@ -549,7 +564,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -605,7 +620,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -631,13 +646,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/129851762.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/283280759.10000.insclause-0 + + 326 + @@ -705,6 +723,9 @@ + + 325 + @@ -782,6 +803,9 @@ + + 324 + Index: ql/src/test/results/compiler/plan/input1.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input1.q.xml (revision 747596) +++ ql/src/test/results/compiler/plan/input1.q.xml (working copy) @@ -31,7 +31,7 @@ true - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/697528137/282404521.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1564028033/350509108.10000.insclause-0 @@ -84,7 +84,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/dest1 + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/dest1 @@ -128,13 +128,16 @@ - /data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/697528137/282404521.10000.insclause-0 + /data/users/njain/hive1/hive/ql/../build/ql/tmp/1564028033/350509108.10000.insclause-0 + + 122 + @@ -206,6 +209,9 @@ + + 121 + @@ -335,6 +341,9 @@ + + 120 + @@ -374,6 +383,9 @@ + + 119 + @@ -388,7 +400,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src src @@ -400,7 +412,7 @@ - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src @@ -456,7 +468,7 @@ location - file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src + file:/data/users/njain/hive1/hive/build/ql/test/data/warehouse/src Index: ql/src/test/queries/clientpositive/groupby7_noskew.q =================================================================== --- ql/src/test/queries/clientpositive/groupby7_noskew.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby7_noskew.q (revision 0) @@ -0,0 +1,22 @@ +set hive.map.aggr=false; + +set hive.groupby.skewindata=false; +set mapred.reduce.tasks=31; + +CREATE TABLE DEST1(key INT, value STRING) STORED AS TEXTFILE; +CREATE TABLE DEST2(key INT, value STRING) STORED AS TEXTFILE; + +SET hive.exec.compress.intermediate=true; +SET hive.exec.compress.output=true; + +EXPLAIN +FROM SRC +INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key +INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key; + +FROM SRC +INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key +INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key; + +SELECT DEST1.* FROM DEST1; +SELECT DEST2.* FROM DEST2; Index: ql/src/test/queries/clientpositive/groupby8_map_skew.q =================================================================== --- ql/src/test/queries/clientpositive/groupby8_map_skew.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby8_map_skew.q (revision 0) @@ -0,0 +1,19 @@ +set hive.map.aggr=true; +set hive.groupby.skewindata=true; +set mapred.reduce.tasks=31; + +CREATE TABLE DEST1(key INT, value STRING) STORED AS TEXTFILE; +CREATE TABLE DEST2(key INT, value STRING) STORED AS TEXTFILE; + +EXPLAIN +FROM SRC +INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key +INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key; + +FROM SRC +INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key +INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key; + +SELECT DEST1.* FROM DEST1; +SELECT DEST2.* FROM DEST2; + Index: ql/src/test/queries/clientpositive/groupby2_map.q =================================================================== --- ql/src/test/queries/clientpositive/groupby2_map.q (revision 747596) +++ ql/src/test/queries/clientpositive/groupby2_map.q (working copy) @@ -1,4 +1,6 @@ set hive.map.aggr=true; +set hive.groupby.skewindata=false; +set mapred.reduce.tasks=31; CREATE TABLE dest1(key STRING, c1 INT, c2 STRING) STORED AS TEXTFILE; Index: ql/src/test/queries/clientpositive/groupby6_map.q =================================================================== --- ql/src/test/queries/clientpositive/groupby6_map.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby6_map.q (revision 0) @@ -0,0 +1,16 @@ +set hive.map.aggr=true; +set hive.groupby.skewindata=false; +set mapred.reduce.tasks=31; + +CREATE TABLE dest1(c1 STRING) STORED AS TEXTFILE; + +EXPLAIN +FROM src +INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1); + +FROM src +INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1); + +SELECT dest1.* FROM dest1; + + Index: ql/src/test/queries/clientpositive/groupby3_map_skew.q =================================================================== --- ql/src/test/queries/clientpositive/groupby3_map_skew.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby3_map_skew.q (revision 0) @@ -0,0 +1,14 @@ +set hive.map.aggr=true; +set hive.groupby.skewindata=true; +set mapred.reduce.tasks=31; + +CREATE TABLE dest1(c1 DOUBLE, c2 DOUBLE, c3 DOUBLE, c4 DOUBLE, c5 DOUBLE) STORED AS TEXTFILE; + +EXPLAIN +FROM src +INSERT OVERWRITE TABLE dest1 SELECT sum(substr(src.value,5)), avg(substr(src.value,5)), avg(DISTINCT substr(src.value,5)), max(substr(src.value,5)), min(substr(src.value,5)); + +FROM src +INSERT OVERWRITE TABLE dest1 SELECT sum(substr(src.value,5)), avg(substr(src.value,5)), avg(DISTINCT substr(src.value,5)), max(substr(src.value,5)), min(substr(src.value,5)); + +SELECT dest1.* FROM dest1; Index: ql/src/test/queries/clientpositive/groupby8_noskew.q =================================================================== --- ql/src/test/queries/clientpositive/groupby8_noskew.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby8_noskew.q (revision 0) @@ -0,0 +1,20 @@ +set hive.map.aggr=false; + +set hive.groupby.skewindata=false; +set mapred.reduce.tasks=31; + +CREATE TABLE DEST1(key INT, value STRING) STORED AS TEXTFILE; +CREATE TABLE DEST2(key INT, value STRING) STORED AS TEXTFILE; + +EXPLAIN +FROM SRC +INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key +INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key; + +FROM SRC +INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key +INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key; + +SELECT DEST1.* FROM DEST1; +SELECT DEST2.* FROM DEST2; + Index: ql/src/test/queries/clientpositive/groupby2.q =================================================================== --- ql/src/test/queries/clientpositive/groupby2.q (revision 747596) +++ ql/src/test/queries/clientpositive/groupby2.q (working copy) @@ -1,3 +1,6 @@ +set hive.map.aggr=false; +set hive.groupby.skewindata=true; + CREATE TABLE dest_g2(key STRING, c1 INT, c2 STRING) STORED AS TEXTFILE; EXPLAIN Index: ql/src/test/queries/clientpositive/groupby6.q =================================================================== --- ql/src/test/queries/clientpositive/groupby6.q (revision 747596) +++ ql/src/test/queries/clientpositive/groupby6.q (working copy) @@ -1,3 +1,6 @@ +set hive.map.aggr=false; +set hive.groupby.skewindata=true; + CREATE TABLE dest1(c1 STRING) STORED AS TEXTFILE; EXPLAIN Index: ql/src/test/queries/clientpositive/groupby1_noskew.q =================================================================== --- ql/src/test/queries/clientpositive/groupby1_noskew.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby1_noskew.q (revision 0) @@ -0,0 +1,12 @@ +set hive.map.aggr=false; +set hive.groupby.skewindata=false; +set mapred.reduce.tasks=31; + +CREATE TABLE dest_g1(key INT, value DOUBLE) STORED AS TEXTFILE; + +EXPLAIN +FROM src INSERT OVERWRITE TABLE dest_g1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key; + +FROM src INSERT OVERWRITE TABLE dest_g1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key; + +SELECT dest_g1.* FROM dest_g1; Index: ql/src/test/queries/clientpositive/groupby6_map_skew.q =================================================================== --- ql/src/test/queries/clientpositive/groupby6_map_skew.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby6_map_skew.q (revision 0) @@ -0,0 +1,16 @@ +set hive.map.aggr=true; +set hive.groupby.skewindata=true; +set mapred.reduce.tasks=31; + +CREATE TABLE dest1(c1 STRING) STORED AS TEXTFILE; + +EXPLAIN +FROM src +INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1); + +FROM src +INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1); + +SELECT dest1.* FROM dest1; + + Index: ql/src/test/queries/clientpositive/groupby1_map.q =================================================================== --- ql/src/test/queries/clientpositive/groupby1_map.q (revision 747596) +++ ql/src/test/queries/clientpositive/groupby1_map.q (working copy) @@ -1,4 +1,6 @@ set hive.map.aggr=true; +set hive.groupby.skewindata=false; +set mapred.reduce.tasks=31; CREATE TABLE dest1(key INT, value DOUBLE) STORED AS TEXTFILE; Index: ql/src/test/queries/clientpositive/groupby5_map.q =================================================================== --- ql/src/test/queries/clientpositive/groupby5_map.q (revision 747596) +++ ql/src/test/queries/clientpositive/groupby5_map.q (working copy) @@ -1,4 +1,6 @@ set hive.map.aggr=true; +set hive.groupby.skewindata=false; +set mapred.reduce.tasks=31; CREATE TABLE dest1(key INT) STORED AS TEXTFILE; Index: ql/src/test/queries/clientpositive/groupby1_map_skew.q =================================================================== --- ql/src/test/queries/clientpositive/groupby1_map_skew.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby1_map_skew.q (revision 0) @@ -0,0 +1,12 @@ +set hive.map.aggr=true; +set hive.groupby.skewindata=true; +set mapred.reduce.tasks=31; + +CREATE TABLE dest1(key INT, value DOUBLE) STORED AS TEXTFILE; + +EXPLAIN +FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key; + +FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key; + +SELECT dest1.* FROM dest1; Index: ql/src/test/queries/clientpositive/groupby2_noskew.q =================================================================== --- ql/src/test/queries/clientpositive/groupby2_noskew.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby2_noskew.q (revision 0) @@ -0,0 +1,14 @@ +set hive.map.aggr=false; +set hive.groupby.skewindata=false; +set mapred.reduce.tasks=31; + +CREATE TABLE dest_g2(key STRING, c1 INT, c2 STRING) STORED AS TEXTFILE; + +EXPLAIN +FROM src +INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1); + +FROM src +INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1); + +SELECT dest_g2.* FROM dest_g2; Index: ql/src/test/queries/clientpositive/groupby3.q =================================================================== --- ql/src/test/queries/clientpositive/groupby3.q (revision 747596) +++ ql/src/test/queries/clientpositive/groupby3.q (working copy) @@ -1,3 +1,6 @@ +set hive.map.aggr=false; +set hive.groupby.skewindata=true; + CREATE TABLE dest1(c1 DOUBLE, c2 DOUBLE, c3 DOUBLE, c4 DOUBLE, c5 DOUBLE) STORED AS TEXTFILE; EXPLAIN Index: ql/src/test/queries/clientpositive/groupby7.q =================================================================== --- ql/src/test/queries/clientpositive/groupby7.q (revision 747596) +++ ql/src/test/queries/clientpositive/groupby7.q (working copy) @@ -1,3 +1,6 @@ +set hive.map.aggr=false; +set hive.groupby.skewindata=true; + CREATE TABLE DEST1(key INT, value STRING) STORED AS TEXTFILE; CREATE TABLE DEST2(key INT, value STRING) STORED AS TEXTFILE; Index: ql/src/test/queries/clientpositive/groupby3_noskew.q =================================================================== --- ql/src/test/queries/clientpositive/groupby3_noskew.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby3_noskew.q (revision 0) @@ -0,0 +1,15 @@ +set hive.map.aggr=false; + +set hive.groupby.skewindata=false; +set mapred.reduce.tasks=31; + +CREATE TABLE dest1(c1 DOUBLE, c2 DOUBLE, c3 DOUBLE, c4 DOUBLE, c5 DOUBLE) STORED AS TEXTFILE; + +EXPLAIN +FROM src +INSERT OVERWRITE TABLE dest1 SELECT sum(substr(src.value,5)), avg(substr(src.value,5)), avg(DISTINCT substr(src.value,5)), max(substr(src.value,5)), min(substr(src.value,5)); + +FROM src +INSERT OVERWRITE TABLE dest1 SELECT sum(substr(src.value,5)), avg(substr(src.value,5)), avg(DISTINCT substr(src.value,5)), max(substr(src.value,5)), min(substr(src.value,5)); + +SELECT dest1.* FROM dest1; Index: ql/src/test/queries/clientpositive/groupby4_map_skew.q =================================================================== --- ql/src/test/queries/clientpositive/groupby4_map_skew.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby4_map_skew.q (revision 0) @@ -0,0 +1,12 @@ +set hive.map.aggr=true; +set hive.groupby.skewindata=true; +set mapred.reduce.tasks=31; + +CREATE TABLE dest1(key INT) STORED AS TEXTFILE; + +EXPLAIN +FROM src INSERT OVERWRITE TABLE dest1 SELECT count(1); + +FROM src INSERT OVERWRITE TABLE dest1 SELECT count(1); + +SELECT dest1.* FROM dest1; Index: ql/src/test/queries/clientpositive/groupby4_map.q =================================================================== --- ql/src/test/queries/clientpositive/groupby4_map.q (revision 747596) +++ ql/src/test/queries/clientpositive/groupby4_map.q (working copy) @@ -1,4 +1,6 @@ set hive.map.aggr=true; +set hive.groupby.skewindata=false; +set mapred.reduce.tasks=31; CREATE TABLE dest1(key INT) STORED AS TEXTFILE; Index: ql/src/test/queries/clientpositive/groupby8_map.q =================================================================== --- ql/src/test/queries/clientpositive/groupby8_map.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby8_map.q (revision 0) @@ -0,0 +1,19 @@ +set hive.map.aggr=true; +set hive.groupby.skewindata=false; +set mapred.reduce.tasks=31; + +CREATE TABLE DEST1(key INT, value STRING) STORED AS TEXTFILE; +CREATE TABLE DEST2(key INT, value STRING) STORED AS TEXTFILE; + +EXPLAIN +FROM SRC +INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key +INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key; + +FROM SRC +INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key +INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key; + +SELECT DEST1.* FROM DEST1; +SELECT DEST2.* FROM DEST2; + Index: ql/src/test/queries/clientpositive/groupby4_noskew.q =================================================================== --- ql/src/test/queries/clientpositive/groupby4_noskew.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby4_noskew.q (revision 0) @@ -0,0 +1,16 @@ +set hive.map.aggr=false; + +set hive.groupby.skewindata=false; +set mapred.reduce.tasks=31; + +CREATE TABLE dest1(c1 STRING) STORED AS TEXTFILE; + +EXPLAIN +FROM src +INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1) GROUP BY substr(src.key,1,1); + +FROM src +INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1) GROUP BY substr(src.key,1,1); + +SELECT dest1.* FROM dest1; + Index: ql/src/test/queries/clientpositive/groupby7_map_skew.q =================================================================== --- ql/src/test/queries/clientpositive/groupby7_map_skew.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby7_map_skew.q (revision 0) @@ -0,0 +1,21 @@ +set hive.map.aggr=true; +set hive.groupby.skewindata=true; +set mapred.reduce.tasks=31; + +CREATE TABLE DEST1(key INT, value STRING) STORED AS TEXTFILE; +CREATE TABLE DEST2(key INT, value STRING) STORED AS TEXTFILE; + +SET hive.exec.compress.intermediate=true; +SET hive.exec.compress.output=true; + +EXPLAIN +FROM SRC +INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key +INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key; + +FROM SRC +INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key +INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key; + +SELECT DEST1.* FROM DEST1; +SELECT DEST2.* FROM DEST2; Index: ql/src/test/queries/clientpositive/groupby4.q =================================================================== --- ql/src/test/queries/clientpositive/groupby4.q (revision 747596) +++ ql/src/test/queries/clientpositive/groupby4.q (working copy) @@ -1,3 +1,6 @@ +set hive.map.aggr=false; +set hive.groupby.skewindata=true; + CREATE TABLE dest1(c1 STRING) STORED AS TEXTFILE; EXPLAIN Index: ql/src/test/queries/clientpositive/groupby8.q =================================================================== --- ql/src/test/queries/clientpositive/groupby8.q (revision 747596) +++ ql/src/test/queries/clientpositive/groupby8.q (working copy) @@ -1,10 +1,18 @@ +set hive.map.aggr=false; +set hive.groupby.skewindata=true; + CREATE TABLE DEST1(key INT, value STRING) STORED AS TEXTFILE; CREATE TABLE DEST2(key INT, value STRING) STORED AS TEXTFILE; +EXPLAIN FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key; +FROM SRC +INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key +INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key; + SELECT DEST1.* FROM DEST1; SELECT DEST2.* FROM DEST2; Index: ql/src/test/queries/clientpositive/groupby2_map_skew.q =================================================================== --- ql/src/test/queries/clientpositive/groupby2_map_skew.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby2_map_skew.q (revision 0) @@ -0,0 +1,14 @@ +set hive.map.aggr=true; +set hive.groupby.skewindata=true; +set mapred.reduce.tasks=31; + +CREATE TABLE dest1(key STRING, c1 INT, c2 STRING) STORED AS TEXTFILE; + +EXPLAIN +FROM src +INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1); + +FROM src +INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1); + +SELECT dest1.* FROM dest1; Index: ql/src/test/queries/clientpositive/groupby5_noskew.q =================================================================== --- ql/src/test/queries/clientpositive/groupby5_noskew.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby5_noskew.q (revision 0) @@ -0,0 +1,20 @@ +set hive.map.aggr=false; + +set hive.groupby.skewindata=false; +set mapred.reduce.tasks=31; + +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; + +EXPLAIN +INSERT OVERWRITE TABLE dest1 +SELECT src.key, sum(substr(src.value,5)) +FROM src +GROUP BY src.key; + +INSERT OVERWRITE TABLE dest1 +SELECT src.key, sum(substr(src.value,5)) +FROM src +GROUP BY src.key; + +SELECT dest1.* FROM dest1; + Index: ql/src/test/queries/clientpositive/groupby3_map.q =================================================================== --- ql/src/test/queries/clientpositive/groupby3_map.q (revision 747596) +++ ql/src/test/queries/clientpositive/groupby3_map.q (working copy) @@ -1,4 +1,6 @@ set hive.map.aggr=true; +set hive.groupby.skewindata=false; +set mapred.reduce.tasks=31; CREATE TABLE dest1(c1 DOUBLE, c2 DOUBLE, c3 DOUBLE, c4 DOUBLE, c5 DOUBLE) STORED AS TEXTFILE; Index: ql/src/test/queries/clientpositive/groupby7_map.q =================================================================== --- ql/src/test/queries/clientpositive/groupby7_map.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby7_map.q (revision 0) @@ -0,0 +1,21 @@ +set hive.map.aggr=true; +set hive.groupby.skewindata=false; +set mapred.reduce.tasks=31; + +CREATE TABLE DEST1(key INT, value STRING) STORED AS TEXTFILE; +CREATE TABLE DEST2(key INT, value STRING) STORED AS TEXTFILE; + +SET hive.exec.compress.intermediate=true; +SET hive.exec.compress.output=true; + +EXPLAIN +FROM SRC +INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key +INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key; + +FROM SRC +INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key +INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key; + +SELECT DEST1.* FROM DEST1; +SELECT DEST2.* FROM DEST2; Index: ql/src/test/queries/clientpositive/groupby5_map_skew.q =================================================================== --- ql/src/test/queries/clientpositive/groupby5_map_skew.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby5_map_skew.q (revision 0) @@ -0,0 +1,12 @@ +set hive.map.aggr=true; +set hive.groupby.skewindata=true; +set mapred.reduce.tasks=31; + +CREATE TABLE dest1(key INT) STORED AS TEXTFILE; + +EXPLAIN +FROM src INSERT OVERWRITE TABLE dest1 SELECT sum(src.key); + +FROM src INSERT OVERWRITE TABLE dest1 SELECT sum(src.key); + +SELECT dest1.* FROM dest1; Index: ql/src/test/queries/clientpositive/groupby6_noskew.q =================================================================== --- ql/src/test/queries/clientpositive/groupby6_noskew.q (revision 0) +++ ql/src/test/queries/clientpositive/groupby6_noskew.q (revision 0) @@ -0,0 +1,17 @@ +set hive.map.aggr=false; + +set hive.groupby.skewindata=false; +set mapred.reduce.tasks=31; + +CREATE TABLE dest1(c1 STRING) STORED AS TEXTFILE; + +EXPLAIN +FROM src +INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1); + +FROM src +INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1); + +SELECT dest1.* FROM dest1; + + Index: ql/src/test/queries/clientpositive/groupby1.q =================================================================== --- ql/src/test/queries/clientpositive/groupby1.q (revision 747596) +++ ql/src/test/queries/clientpositive/groupby1.q (working copy) @@ -1,3 +1,6 @@ +set hive.map.aggr=false; +set hive.groupby.skewindata=true; + CREATE TABLE dest_g1(key INT, value DOUBLE) STORED AS TEXTFILE; EXPLAIN Index: ql/src/test/queries/clientpositive/groupby5.q =================================================================== --- ql/src/test/queries/clientpositive/groupby5.q (revision 747596) +++ ql/src/test/queries/clientpositive/groupby5.q (working copy) @@ -1,3 +1,6 @@ +set hive.map.aggr=false; +set hive.groupby.skewindata=true; + CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; EXPLAIN Index: ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (revision 747596) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (working copy) @@ -49,10 +49,11 @@ */ public class GroupByOperator extends Operator implements Serializable { - static final private Log LOG = LogFactory.getLog(JoinOperator.class.getName()); + static final private Log LOG = LogFactory.getLog(GroupByOperator.class.getName()); private static final long serialVersionUID = 1L; private static final int NUMROWSESTIMATESIZE = 1000; + private static final int NUMROWSCHECKHASHAGGR = 100000; transient protected ExprNodeEvaluator[] keyFields; transient protected ExprNodeEvaluator[][] aggregationParameterFields; @@ -78,6 +79,8 @@ transient boolean firstRow; transient long totalMemory; transient boolean hashAggr; + transient long numRowsInput; + transient long numRowsHashTbl; /** * This is used to store the position and field names for variable length fields. @@ -114,13 +117,15 @@ public void initialize(Configuration hconf, Reporter reporter) throws HiveException { super.initialize(hconf, reporter); totalMemory = Runtime.getRuntime().totalMemory(); + numRowsInput = 0; + numRowsHashTbl = 0; // init keyFields keyFields = new ExprNodeEvaluator[conf.getKeys().size()]; for (int i = 0; i < keyFields.length; i++) { keyFields[i] = ExprNodeEvaluatorFactory.get(conf.getKeys().get(i)); } - + // init aggregationParameterFields aggregationParameterFields = new ExprNodeEvaluator[conf.getAggregators().size()][]; for (int i = 0; i < aggregationParameterFields.length; i++) { @@ -186,6 +191,7 @@ hashAggr = false; } else { hashAggregations = new HashMap, UDAFEvaluator[]>(); + aggregations = newAggregations(); hashAggr = true; keyPositionsSize = new ArrayList(); aggrPositions = new ArrayList(); @@ -393,7 +399,22 @@ } public void process(Object row, ObjectInspector rowInspector) throws HiveException { - + // Total number of input rows is needed for hash aggregation only + if (hashAggr) { + numRowsInput++; + // if hash aggregation is not behvaing properly, disable it + if ((numRowsInput % NUMROWSCHECKHASHAGGR) == 0) { + // map-side aggregation should reduce the entries by atleast half + if ((numRowsHashTbl * 2) > numRowsInput) { + LOG.trace("Disable Hash Aggr: #hash table = " + numRowsHashTbl + " #total = " + numRowsInput); + flush(true); + hashAggr = false; + } + else + LOG.trace("Hash Aggr Enabled: #hash table = " + numRowsHashTbl + " #total = " + numRowsInput); + } + } + try { // Compute the keys ArrayList newKeys = new ArrayList(keyFields.length); @@ -436,6 +457,7 @@ aggs = newAggregations(); hashAggregations.put(newKeys, aggs); newEntry = true; + numRowsHashTbl++; // new entry in the hash table } // Update the aggs @@ -443,7 +465,7 @@ // based on used-specified pramaters, check if the hash table needs to be flushed if (shouldBeFlushed(newKeys)) { - flush(); + flush(false); } } @@ -522,12 +544,24 @@ return false; } - private void flush() throws HiveException { + private void flush(boolean complete) throws HiveException { // Currently, the algorithm flushes 10% of the entries - this can be // changed in the future + if (complete) { + Iterator iter = hashAggregations.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry, UDAFEvaluator[]> m = (Map.Entry)iter.next(); + forward(m.getKey(), m.getValue()); + } + hashAggregations.clear(); + hashAggregations = null; + return; + } + int oldSize = hashAggregations.size(); + LOG.trace("Hash Tbl flush: #hash table = " + oldSize); Iterator iter = hashAggregations.entrySet().iterator(); int numDel = 0; while (iter.hasNext()) { @@ -571,16 +605,19 @@ public void close(boolean abort) throws HiveException { if (!abort) { try { - if (aggregations != null) { + if (hashAggregations != null) { + // hash-based aggregations + for (ArrayList key: hashAggregations.keySet()) { + forward(key, hashAggregations.get(key)); + } + hashAggregations.clear(); + } + else if (aggregations != null) { // sort-based aggregations if (currentKeys != null) { forward(currentKeys, aggregations); } - } else if (hashAggregations != null) { - // hash-based aggregations - for (ArrayList key: hashAggregations.keySet()) { - forward(key, hashAggregations.get(key)); - } + currentKeys = null; } else { // The GroupByOperator is not initialized, which means there is no data // (since we initialize the operators when we see the first record). Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (revision 747596) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (working copy) @@ -47,15 +47,23 @@ protected List> childOperators; protected List> parentOperators; + private static int seqId; - public Operator() {} + static { + seqId = 0; + } + public Operator() { + id = String.valueOf(seqId++); + } + /** * Create an operator with a reporter. * @param reporter Used to report progress of certain operators. */ public Operator(Reporter reporter) { this.reporter = reporter; + id = String.valueOf(seqId++); } public void setChildOperators(List> childOperators) { @@ -262,6 +270,7 @@ for(Operator op: childOperators) { op.close(abort); } + } catch (HiveException e) { } } @@ -341,4 +350,27 @@ return new String("OP"); } + public String dump() { + StringBuilder s = new StringBuilder(); + s.append("<" + getName() + ">"); + s.append("Id =" + getId()); + if (childOperators != null) { + s.append(""); + for (Operator o : childOperators) { + s.append(o.dump()); + } + s.append("<\\Children>"); + } + + if (parentOperators != null) { + s.append(""); + for (Operator o : parentOperators) { + s.append("Id = " + o.getId() + " "); + } + s.append("<\\Parent>"); + } + + s.append("<\\" + getName() + ">"); + return s.toString(); + } } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 747596) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -1233,7 +1233,8 @@ } r.aggMethod = null; - String funcName = (((mode == groupByDesc.Mode.PARTIAL1) || (mode == groupByDesc.Mode.HASH)) ? "iterate" : "merge"); + String funcName = (((mode == groupByDesc.Mode.PARTIAL1) || (mode == groupByDesc.Mode.HASH) || + (mode == groupByDesc.Mode.COMPLETE)) ? "iterate" : "merge"); if (aggTree.getToken().getType() == HiveParser.TOK_FUNCTIONDI && (mode != groupByDesc.Mode.FINAL)) funcName = "iterate"; @@ -1555,94 +1556,6 @@ } /** - * Generate the ReduceSinkOperator for the Group By Query Block (parseInfo.getXXX(dest)). - * The new ReduceSinkOperator will be a child of inputOperatorInfo. - * - * It will put all Group By keys and the distinct field (if any) in the map-reduce sort key, - * and all other fields in the map-reduce value. - * - * The map-reduce partition key will be random() if there is no distinct, or the same as - * the map-reduce sort key otherwise. - * - * @return the new ReduceSinkOperator. - * @throws SemanticException - */ - @SuppressWarnings("nls") - private Operator genGroupByPlanReduceSinkOperator(QBParseInfo parseInfo, - String dest, Operator inputOperatorInfo) - throws SemanticException { - - return genGroupByPlanReduceSinkOperator(parseInfo, dest, inputOperatorInfo, -1); - } - - @SuppressWarnings("nls") - private Operator genGroupByPlanReduceSinkOperator(QBParseInfo parseInfo, - String dest, Operator inputOperatorInfo, int numReducers) - throws SemanticException { - RowResolver reduceSinkInputRowResolver = opParseCtx.get(inputOperatorInfo).getRR(); - RowResolver reduceSinkOutputRowResolver = new RowResolver(); - reduceSinkOutputRowResolver.setIsExprResolver(true); - ArrayList reduceKeys = new ArrayList(); - - // Pre-compute group-by keys and store in reduceKeys - List grpByExprs = getGroupByForClause(parseInfo, dest); - for (int i = 0; i < grpByExprs.size(); ++i) { - ASTNode grpbyExpr = grpByExprs.get(i); - String text = grpbyExpr.toStringTree(); - - if (reduceSinkOutputRowResolver.get("", text) == null) { - ColumnInfo exprInfo = reduceSinkInputRowResolver.get("", text); - reduceKeys.add(new exprNodeColumnDesc(exprInfo.getType(), exprInfo.getInternalName())); - reduceSinkOutputRowResolver.put("", text, - new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + Integer.valueOf(reduceKeys.size() - 1).toString(), - exprInfo.getType())); - } - } - - // If there is a distinctFuncExp, add all parameters to the reduceKeys. - if (parseInfo.getDistinctFuncExprForClause(dest) != null) { - ASTNode value = parseInfo.getDistinctFuncExprForClause(dest); - // 0 is function name - for (int i = 1; i < value.getChildCount(); i++) { - ASTNode parameter = (ASTNode) value.getChild(i); - String text = parameter.toStringTree(); - if (reduceSinkOutputRowResolver.get("",text) == null) { - ColumnInfo exprInfo = reduceSinkInputRowResolver.get("", text); - reduceKeys.add(new exprNodeColumnDesc(exprInfo.getType(), exprInfo.getInternalName())); - reduceSinkOutputRowResolver.put("", text, - new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + Integer.valueOf(reduceKeys.size() - 1).toString(), - exprInfo.getType())); - } - } - } - - // Put partial aggregation results in reduceValues - ArrayList reduceValues = new ArrayList(); - HashMap aggregationTrees = parseInfo - .getAggregationExprsForClause(dest); - int inputField = reduceKeys.size(); - - for (Map.Entry entry : aggregationTrees.entrySet()) { - - TypeInfo type = reduceSinkInputRowResolver.getColumnInfos().get(inputField).getType(); - reduceValues.add(new exprNodeColumnDesc( - type, (Integer.valueOf(inputField)).toString())); - inputField++; - reduceSinkOutputRowResolver.put("", ((ASTNode)entry.getValue()).toStringTree(), - new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + (Integer.valueOf(reduceValues.size()-1)).toString(), - type)); - } - - return putOpInsertMap( - OperatorFactory.getAndMakeChild( - PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, -1, - (parseInfo.getDistinctFuncExprForClause(dest) == null ? -1 : Integer.MAX_VALUE), numReducers), - new RowSchema(reduceSinkOutputRowResolver.getColumnInfos()), - inputOperatorInfo), - reduceSinkOutputRowResolver); - } - - /** * Generate the ReduceSinkOperator for the Group By Query Block (qb.getPartInfo().getXXX(dest)). * The new ReduceSinkOperator will be a child of inputOperatorInfo. * @@ -1656,7 +1569,8 @@ */ @SuppressWarnings("nls") private Operator genGroupByPlanReduceSinkOperator(QB qb, - String dest, Operator inputOperatorInfo, int numPartitionFields) throws SemanticException { + String dest, Operator inputOperatorInfo, int numPartitionFields, int numReducers, boolean mapAggrDone) throws SemanticException { + RowResolver reduceSinkInputRowResolver = opParseCtx.get(inputOperatorInfo).getRR(); QBParseInfo parseInfo = qb.getParseInfo(); RowResolver reduceSinkOutputRowResolver = new RowResolver(); @@ -1694,28 +1608,44 @@ } } - // Put parameters to aggregations in reduceValues ArrayList reduceValues = new ArrayList(); - HashMap aggregationTrees = parseInfo - .getAggregationExprsForClause(dest); - for (Map.Entry entry : aggregationTrees.entrySet()) { + HashMap aggregationTrees = parseInfo.getAggregationExprsForClause(dest); + + if (!mapAggrDone) { + // Put parameters to aggregations in reduceValues + for (Map.Entry entry : aggregationTrees.entrySet()) { ASTNode value = entry.getValue(); - // 0 is function name - for (int i = 1; i < value.getChildCount(); i++) { - ASTNode parameter = (ASTNode) value.getChild(i); - String text = parameter.toStringTree(); - if (reduceSinkOutputRowResolver.get("",text) == null) { - reduceValues.add(genExprNodeDesc(parameter, reduceSinkInputRowResolver)); - reduceSinkOutputRowResolver.put("", text, - new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(), - reduceValues.get(reduceValues.size()-1).getTypeInfo())); + // 0 is function name + for (int i = 1; i < value.getChildCount(); i++) { + ASTNode parameter = (ASTNode) value.getChild(i); + String text = parameter.toStringTree(); + if (reduceSinkOutputRowResolver.get("",text) == null) { + reduceValues.add(genExprNodeDesc(parameter, reduceSinkInputRowResolver)); + reduceSinkOutputRowResolver.put("", text, + new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(), + reduceValues.get(reduceValues.size()-1).getTypeInfo())); + } } } } + else + { + // Put partial aggregation results in reduceValues + int inputField = reduceKeys.size(); + + for (Map.Entry entry : aggregationTrees.entrySet()) { + + TypeInfo type = reduceSinkInputRowResolver.getColumnInfos().get(inputField).getType(); + reduceValues.add(new exprNodeColumnDesc(type, (Integer.valueOf(inputField)).toString())); + inputField++; + reduceSinkOutputRowResolver.put("", ((ASTNode)entry.getValue()).toStringTree(), + new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + (Integer.valueOf(reduceValues.size()-1)).toString(), + type)); + } + } return putOpInsertMap( - OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, -1, numPartitionFields, - -1), + OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, -1, numPartitionFields, numReducers), new RowSchema(reduceSinkOutputRowResolver.getColumnInfos()), inputOperatorInfo), reduceSinkOutputRowResolver @@ -1875,9 +1805,8 @@ // ////// 1. Generate ReduceSinkOperator Operator reduceSinkOperatorInfo = genGroupByPlanReduceSinkOperator( - qb, dest, input, getGroupByForClause(parseInfo, dest).size()); + qb, dest, input, getGroupByForClause(parseInfo, dest).size(), -1, false); - // ////// 2. Generate GroupbyOperator Operator groupByOperatorInfo = genGroupByPlanGroupByOperator(parseInfo, dest, reduceSinkOperatorInfo, groupByDesc.Mode.COMPLETE); @@ -1915,7 +1844,7 @@ // captured by WritableComparableHiveObject.hashCode() function. Operator reduceSinkOperatorInfo = genGroupByPlanReduceSinkOperator( qb, dest, input, (parseInfo.getDistinctFuncExprForClause(dest) == null ? -1 - : Integer.MAX_VALUE)); + : Integer.MAX_VALUE), -1, false); // ////// 2. Generate GroupbyOperator Operator groupByOperatorInfo = genGroupByPlanGroupByOperator(parseInfo, @@ -1952,7 +1881,7 @@ * Evaluate partial aggregates first, followed by actual aggregates. */ @SuppressWarnings("nls") - private Operator genGroupByPlan4MR(String dest, QB qb, + private Operator genGroupByPlanMapAggr1MR(String dest, QB qb, Operator inputOperatorInfo) throws SemanticException { QBParseInfo parseInfo = qb.getParseInfo(); @@ -1961,13 +1890,46 @@ Operator groupByOperatorInfo = genGroupByPlanMapGroupByOperator(qb, dest, inputOperatorInfo, groupByDesc.Mode.HASH); + int numReducers = -1; + + // Optimize the scenario when there are no grouping keys and no distinct - only 1 reducer is needed + // For eg: select count(1) from T where t.ds = .... + if (optimizeMapAggrGroupBy(dest, qb)) + numReducers = 1; + + // ////// Generate ReduceSink Operator + Operator reduceSinkOperatorInfo = + genGroupByPlanReduceSinkOperator(qb, dest, groupByOperatorInfo, getGroupByForClause(parseInfo, dest).size(), numReducers, true); + + return genGroupByPlanGroupByOperator2MR(parseInfo, dest, reduceSinkOperatorInfo, groupByDesc.Mode.FINAL); + } + + /** + * Generate a Group-By plan using a 2 map-reduce jobs. First perform a map + * side partial aggregation (to reduce the amount of data). Then spray by + * the distinct key (or a random number) in hope of getting a uniform + * distribution, and compute partial aggregates grouped by that distinct key. + * Evaluate partial aggregates first, followed by actual aggregates. + */ + @SuppressWarnings("nls") + private Operator genGroupByPlanMapAggr2MR(String dest, QB qb, + Operator inputOperatorInfo) throws SemanticException { + + QBParseInfo parseInfo = qb.getParseInfo(); + + // ////// Generate GroupbyOperator for a map-side partial aggregation + Operator groupByOperatorInfo = genGroupByPlanMapGroupByOperator(qb, + dest, inputOperatorInfo, groupByDesc.Mode.HASH); + // Optimize the scenario when there are no grouping keys and no distinct - 2 map-reduce jobs are not needed // For eg: select count(1) from T where t.ds = .... if (!optimizeMapAggrGroupBy(dest, qb)) { // ////// Generate ReduceSink Operator Operator reduceSinkOperatorInfo = - genGroupByPlanReduceSinkOperator(parseInfo, dest, groupByOperatorInfo); + genGroupByPlanReduceSinkOperator(qb, dest, groupByOperatorInfo, + (parseInfo.getDistinctFuncExprForClause(dest) == null ? -1 + : Integer.MAX_VALUE), -1, true); // ////// Generate GroupbyOperator for a partial aggregation Operator groupByOperatorInfo2 = genGroupByPlanGroupByOperator1(parseInfo, dest, reduceSinkOperatorInfo, @@ -1983,7 +1945,7 @@ else { // ////// Generate ReduceSink Operator Operator reduceSinkOperatorInfo = - genGroupByPlanReduceSinkOperator(parseInfo, dest, groupByOperatorInfo, 1); + genGroupByPlanReduceSinkOperator(qb, dest, groupByOperatorInfo, getGroupByForClause(parseInfo, dest).size(), 1, true); return genGroupByPlanGroupByOperator2MR(parseInfo, dest, reduceSinkOperatorInfo, groupByDesc.Mode.FINAL); } @@ -2740,10 +2702,16 @@ if (qbp.getAggregationExprsForClause(dest).size() != 0 || getGroupByForClause(qbp, dest).size() > 0) { - if (conf.getVar(HiveConf.ConfVars.HIVEMAPSIDEAGGREGATE).equalsIgnoreCase("true")) - curr = genGroupByPlan4MR(dest, qb, curr); - else + if (conf.getVar(HiveConf.ConfVars.HIVEMAPSIDEAGGREGATE).equalsIgnoreCase("true")) { + if (conf.getVar(HiveConf.ConfVars.HIVEGROUPBYSKEW).equalsIgnoreCase("false")) + curr = genGroupByPlanMapAggr1MR(dest, qb, curr); + else + curr = genGroupByPlanMapAggr2MR(dest, qb, curr); + } + else if (conf.getVar(HiveConf.ConfVars.HIVEGROUPBYSKEW).equalsIgnoreCase("true")) curr = genGroupByPlan2MR(dest, qb, curr); + else + curr = genGroupByPlan1MR(dest, qb, curr); } curr = genSelectPlan(dest, qb, curr); @@ -2833,6 +2801,7 @@ parent.add(leftOp); parent.add(rightOp); unionforward.setParentOperators(parent); + // create operator info list to return return putOpInsertMap(unionforward, unionoutRR); } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java (revision 747596) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java (working copy) @@ -54,9 +54,15 @@ public boolean merge(String o) { if (o != null && !o.isEmpty()) { int pos = o.indexOf('/'); - assert(pos != -1); - mSum += Double.parseDouble(o.substring(0, pos)); - mCount += Long.parseLong(o.substring(pos+1)); + // In case merge is called without an iterate, the + if (pos == -1) { + mSum += Double.parseDouble(o); + mCount ++; + } + else { + mSum += Double.parseDouble(o.substring(0, pos)); + mCount += Long.parseLong(o.substring(pos+1)); + } } return true; }