Index: data/conf/tez/hive-site.xml =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/xml Index: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java (revision 1667454) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java (working copy) @@ -151,6 +151,8 @@ this.conf = conf; this.queueName = conf.get("tez.queue.name"); this.doAsEnabled = conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS); + + final boolean llapMode = "llap".equals(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_MODE)); UserGroupInformation ugi = Utils.getUGI(); user = ugi.getShortUserName(); @@ -179,7 +181,7 @@ commonLocalResources.put(utils.getBaseName(lr), lr); } - if ("llap".equals(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_MODE))) { + if (llapMode) { // add configs for llap-daemon-site.xml + localize llap jars // they cannot be referred to directly as it would be a circular depedency conf.addResource("llap-daemon-site.xml"); @@ -217,16 +219,27 @@ // set up the staging directory to use tezConfig.set(TezConfiguration.TEZ_AM_STAGING_DIR, tezScratchDir.toUri().toString()); - // we need plugins to handle llap and uber mode - tezConfig.setStrings(TezConfiguration.TEZ_AM_TASK_SCHEDULERS, - DEFAULT_SERVICE, LOCAL_SERVICE, LLAP_SERVICE_SCHEDULER); + if (llapMode) { + // we need plugins to handle llap and uber mode + tezConfig.setStrings(TezConfiguration.TEZ_AM_TASK_SCHEDULERS, DEFAULT_SERVICE, LOCAL_SERVICE, + LLAP_SERVICE_SCHEDULER); - tezConfig.setStrings(TezConfiguration.TEZ_AM_CONTAINER_LAUNCHERS, - DEFAULT_SERVICE, LOCAL_SERVICE, LLAP_SERVICE_LAUNCHER); + tezConfig.setStrings(TezConfiguration.TEZ_AM_CONTAINER_LAUNCHERS, DEFAULT_SERVICE, + LOCAL_SERVICE, LLAP_SERVICE_LAUNCHER); - tezConfig.setStrings(TezConfiguration.TEZ_AM_TASK_COMMUNICATORS, - DEFAULT_SERVICE, LOCAL_SERVICE, LLAP_SERVICE_TASK_COMMUNICATOR); + tezConfig.setStrings(TezConfiguration.TEZ_AM_TASK_COMMUNICATORS, DEFAULT_SERVICE, + LOCAL_SERVICE, LLAP_SERVICE_TASK_COMMUNICATOR); + } else { + // we need plugins to handle llap and uber mode + tezConfig.setStrings(TezConfiguration.TEZ_AM_TASK_SCHEDULERS, DEFAULT_SERVICE, LOCAL_SERVICE); + tezConfig.setStrings(TezConfiguration.TEZ_AM_CONTAINER_LAUNCHERS, DEFAULT_SERVICE, + LOCAL_SERVICE); + + tezConfig.setStrings(TezConfiguration.TEZ_AM_TASK_COMMUNICATORS, DEFAULT_SERVICE, + LOCAL_SERVICE); + } + // container prewarming. tell the am how many containers we need if (HiveConf.getBoolVar(conf, ConfVars.HIVE_PREWARM_ENABLED)) { int n = HiveConf.getIntVar(conf, ConfVars.HIVE_PREWARM_NUM_CONTAINERS); Index: ql/src/test/queries/clientpositive/llapdecider.q =================================================================== --- ql/src/test/queries/clientpositive/llapdecider.q (revision 1667454) +++ ql/src/test/queries/clientpositive/llapdecider.q (working copy) @@ -1,4 +1,5 @@ set hive.stats.fetch.column.stats=true; +set hive.execution.mode=llap; set hive.llap.execution.mode=auto; -- simple query with multiple reduce stages @@ -58,3 +59,5 @@ EXPLAIN SELECT sum(cast(key as int) + 1) from src_orc where cast(key as int) > 1; EXPLAIN SELECT sum(cast(test_udf_get_java_string(cast(key as string)) as int) + 1) from src_orc where cast(key as int) > 1; EXPLAIN SELECT sum(cast(key as int) + 1) from src_orc where cast(test_udf_get_java_string(cast(key as string)) as int) > 1; + +set hive.execution.mode=container; \ No newline at end of file Index: ql/src/test/queries/clientpositive/mrr.q =================================================================== --- ql/src/test/queries/clientpositive/mrr.q (revision 1667454) +++ ql/src/test/queries/clientpositive/mrr.q (working copy) @@ -1,16 +1,16 @@ -- simple query with multiple reduce stages -EXPLAIN SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt; -SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt; +EXPLAIN SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt,key; +SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt,key; set hive.auto.convert.join=false; -- join query with multiple reduce stages; -EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt; -SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt; +EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt,s2.key; +SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt,s2.key; set hive.auto.convert.join=true; -- same query with broadcast join -EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt; -SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt; +EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt,s2.key; +SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt,s2.key; set hive.auto.convert.join=false; -- query with multiple branches in the task dag Index: ql/src/test/results/clientpositive/tez/mrr.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/mrr.q.out (revision 1667454) +++ ql/src/test/results/clientpositive/tez/mrr.q.out (working copy) @@ -1,8 +1,8 @@ PREHOOK: query: -- simple query with multiple reduce stages -EXPLAIN SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt +EXPLAIN SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt,key PREHOOK: type: QUERY POSTHOOK: query: -- simple query with multiple reduce stages -EXPLAIN SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt +EXPLAIN SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt,key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -46,14 +46,13 @@ outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: bigint) - sort order: + + key expressions: _col1 (type: bigint), _col0 (type: string) + sort order: ++ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) Reducer 3 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint) + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -70,283 +69,297 @@ Processor Tree: ListSink -PREHOOK: query: SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt +PREHOOK: query: SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt,key PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt +POSTHOOK: query: SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt,key POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -490 1 -287 1 -286 1 -285 1 -284 1 -283 1 +10 1 +105 1 +11 1 +111 1 114 1 -487 1 -485 1 -28 1 -484 1 -181 1 -275 1 -274 1 -183 1 -483 1 -27 1 -266 1 -482 1 -263 1 -262 1 -260 1 -481 1 -258 1 -257 1 116 1 -479 1 -252 1 -249 1 -248 1 -247 1 -244 1 -92 1 -241 1 -477 1 -475 1 -472 1 -470 1 -235 1 -47 1 -186 1 126 1 -228 1 -226 1 131 1 -467 1 -222 1 133 1 -82 1 -218 1 -80 1 -460 1 -214 1 -8 1 -78 1 -189 1 -457 1 -455 1 136 1 -202 1 -201 1 -453 1 -20 1 -2 1 -19 1 -452 1 -196 1 -449 1 -194 1 -190 1 -192 1 -448 1 -446 1 -444 1 -443 1 -44 1 -77 1 143 1 -437 1 -436 1 -435 1 -432 1 145 1 150 1 -43 1 -10 1 -427 1 -74 1 -421 1 -9 1 -419 1 -418 1 153 1 -105 1 -69 1 -411 1 -41 1 155 1 -407 1 156 1 -87 1 157 1 -402 1 158 1 -400 1 -4 1 -66 1 -65 1 160 1 -64 1 -394 1 -393 1 -392 1 -389 1 -386 1 162 1 -86 1 -379 1 -378 1 -377 1 -375 1 -374 1 -373 1 -57 1 163 1 -368 1 -54 1 -366 1 -365 1 -364 1 -362 1 -360 1 -356 1 -53 1 -351 1 166 1 168 1 -345 1 -85 1 -11 1 -341 1 -34 1 -339 1 -338 1 -336 1 -335 1 -111 1 -332 1 -497 1 -33 1 17 1 -496 1 -323 1 -495 1 -494 1 170 1 -493 1 177 1 -315 1 178 1 +180 1 +181 1 +183 1 +186 1 +189 1 +19 1 +190 1 +192 1 +194 1 +196 1 +2 1 +20 1 +201 1 +202 1 +214 1 +218 1 +222 1 +226 1 +228 1 +235 1 +241 1 +244 1 +247 1 +248 1 +249 1 +252 1 +257 1 +258 1 +260 1 +262 1 +263 1 +266 1 +27 1 +274 1 +275 1 +28 1 +283 1 +284 1 +285 1 +286 1 +287 1 +289 1 +291 1 +292 1 +296 1 +30 1 +302 1 +305 1 +306 1 +308 1 310 1 +315 1 +323 1 +33 1 +332 1 +335 1 +336 1 +338 1 +339 1 +34 1 +341 1 +345 1 +351 1 +356 1 +360 1 +362 1 +364 1 +365 1 +366 1 +368 1 +373 1 +374 1 +375 1 +377 1 +378 1 +379 1 +386 1 +389 1 +392 1 +393 1 +394 1 +4 1 +400 1 +402 1 +407 1 +41 1 +411 1 +418 1 +419 1 +421 1 +427 1 +43 1 +432 1 +435 1 +436 1 +437 1 +44 1 +443 1 +444 1 +446 1 +448 1 +449 1 +452 1 +453 1 +455 1 +457 1 +460 1 +467 1 +47 1 +470 1 +472 1 +475 1 +477 1 +479 1 +481 1 +482 1 +483 1 +484 1 +485 1 +487 1 +490 1 +491 1 +493 1 +494 1 +495 1 +496 1 +497 1 +53 1 +54 1 +57 1 +64 1 +65 1 +66 1 +69 1 +74 1 +77 1 +78 1 +8 1 +80 1 +82 1 +85 1 +86 1 +87 1 +9 1 +92 1 96 1 -308 1 -491 1 -306 1 -305 1 -302 1 -30 1 -180 1 -296 1 -292 1 -291 1 -289 1 +100 2 +103 2 +104 2 +113 2 +118 2 +12 2 +120 2 +125 2 +129 2 +134 2 +137 2 +146 2 +149 2 +15 2 +152 2 +164 2 +165 2 +172 2 +174 2 +175 2 +176 2 +179 2 +18 2 +191 2 +195 2 +197 2 +200 2 +203 2 +205 2 +207 2 +209 2 +213 2 +216 2 +217 2 +219 2 +221 2 +223 2 +224 2 +229 2 +233 2 +237 2 +238 2 +239 2 +24 2 +242 2 +255 2 +256 2 +26 2 +265 2 +272 2 +278 2 +280 2 +281 2 +282 2 +288 2 +307 2 +309 2 +317 2 +321 2 +322 2 +325 2 +331 2 +333 2 +342 2 +344 2 +353 2 +367 2 +37 2 +382 2 +395 2 +397 2 +399 2 +404 2 +413 2 +414 2 +42 2 +424 2 +429 2 +439 2 +458 2 +459 2 +462 2 +463 2 +478 2 +492 2 +51 2 +58 2 +67 2 +72 2 +76 2 +83 2 +84 2 +95 2 +97 2 98 2 -97 2 -95 2 -84 2 -83 2 -76 2 -72 2 -67 2 -58 2 -51 2 -492 2 -478 2 -463 2 -462 2 -459 2 -458 2 -439 2 -429 2 -424 2 -42 2 -414 2 -413 2 -404 2 -399 2 -397 2 -395 2 -382 2 -37 2 -367 2 -353 2 -344 2 -342 2 -333 2 -331 2 -325 2 -322 2 -321 2 -317 2 -309 2 -307 2 -288 2 -282 2 -281 2 -280 2 -278 2 -272 2 -265 2 -26 2 -256 2 -255 2 -242 2 -24 2 -239 2 -238 2 -237 2 -233 2 -229 2 -224 2 -223 2 -221 2 -219 2 -217 2 -216 2 -213 2 -209 2 -207 2 -205 2 -203 2 -200 2 -197 2 -195 2 -191 2 -18 2 -179 2 -176 2 -175 2 -174 2 -172 2 -165 2 -164 2 -152 2 -15 2 -149 2 -146 2 -137 2 -134 2 -129 2 -125 2 -120 2 -12 2 -118 2 -113 2 -104 2 -103 2 -100 2 -498 3 +0 3 +119 3 +128 3 +167 3 +187 3 +193 3 +199 3 +208 3 +273 3 +298 3 +311 3 +316 3 +318 3 +327 3 +35 3 369 3 384 3 396 3 @@ -353,45 +366,31 @@ 403 3 409 3 417 3 -5 3 430 3 -70 3 -119 3 -0 3 431 3 438 3 -480 3 -193 3 -199 3 -208 3 -187 3 -273 3 -298 3 454 3 -311 3 -316 3 466 3 +480 3 +498 3 +5 3 +70 3 90 3 -128 3 -318 3 -327 3 -167 3 -35 3 +138 4 +169 4 +277 4 +406 4 468 4 489 4 -406 4 -169 4 -138 4 -277 4 -469 5 -401 5 230 5 348 5 +401 5 +469 5 PREHOOK: query: -- join query with multiple reduce stages; -EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt +EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt,s2.key PREHOOK: type: QUERY POSTHOOK: query: -- join query with multiple reduce stages; -EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt +EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt,s2.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -471,14 +470,13 @@ outputColumnNames: _col0, _col1 Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: bigint) - sort order: + + key expressions: _col1 (type: bigint), _col0 (type: string) + sort order: ++ Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) Reducer 4 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint) + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -495,328 +493,328 @@ Processor Tree: ListSink -PREHOOK: query: SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt +PREHOOK: query: SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt,s2.key PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt +POSTHOOK: query: SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt,s2.key POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +0 1 +10 1 +100 1 +103 1 +104 1 +105 1 +11 1 +111 1 +113 1 +114 1 +116 1 +118 1 +119 1 +12 1 +120 1 +125 1 +126 1 +128 1 +129 1 +131 1 +133 1 +134 1 +136 1 +137 1 +138 1 +143 1 +145 1 +146 1 +149 1 +15 1 +150 1 +152 1 +153 1 +155 1 +156 1 +157 1 +158 1 +160 1 +162 1 +163 1 +164 1 +165 1 +166 1 +167 1 +168 1 +169 1 +17 1 +170 1 +172 1 +174 1 +175 1 +176 1 +177 1 +178 1 +179 1 +18 1 +180 1 +181 1 +183 1 +186 1 +187 1 +189 1 +19 1 +190 1 +191 1 +192 1 +193 1 +194 1 +195 1 +196 1 +197 1 +199 1 +2 1 +20 1 +200 1 +201 1 +202 1 +203 1 +205 1 +207 1 +208 1 +209 1 +213 1 +214 1 +216 1 +217 1 +218 1 +219 1 +221 1 +222 1 +223 1 +224 1 +226 1 +228 1 +229 1 +230 1 +233 1 +235 1 +237 1 +238 1 +239 1 +24 1 +241 1 +242 1 +244 1 +247 1 +248 1 +249 1 +252 1 +255 1 +256 1 +257 1 +258 1 +26 1 +260 1 +262 1 +263 1 +265 1 +266 1 +27 1 +272 1 +273 1 +274 1 +275 1 +277 1 +278 1 +28 1 +280 1 +281 1 +282 1 +283 1 +284 1 +285 1 +286 1 +287 1 +288 1 +289 1 +291 1 +292 1 +296 1 +298 1 +30 1 +302 1 +305 1 +306 1 +307 1 +308 1 +309 1 +310 1 +311 1 +315 1 +316 1 +317 1 +318 1 +321 1 +322 1 +323 1 +325 1 +327 1 +33 1 +331 1 +332 1 +333 1 +335 1 +336 1 +338 1 +339 1 +34 1 +341 1 +342 1 +344 1 +345 1 +348 1 +35 1 +351 1 +353 1 +356 1 +360 1 +362 1 +364 1 +365 1 +366 1 +367 1 +368 1 +369 1 +37 1 +373 1 +374 1 +375 1 +377 1 +378 1 +379 1 +382 1 +384 1 +386 1 +389 1 +392 1 +393 1 +394 1 +395 1 +396 1 +397 1 +399 1 +4 1 +400 1 +401 1 +402 1 +403 1 +404 1 +406 1 +407 1 +409 1 +41 1 +411 1 +413 1 +414 1 +417 1 +418 1 +419 1 +42 1 +421 1 +424 1 +427 1 +429 1 +43 1 +430 1 +431 1 +432 1 +435 1 +436 1 +437 1 +438 1 +439 1 +44 1 +443 1 +444 1 +446 1 +448 1 +449 1 +452 1 +453 1 +454 1 +455 1 +457 1 +458 1 +459 1 +460 1 +462 1 +463 1 +466 1 +467 1 +468 1 +469 1 +47 1 +470 1 +472 1 +475 1 +477 1 +478 1 +479 1 +480 1 +481 1 +482 1 +483 1 +484 1 +485 1 +487 1 +489 1 +490 1 +491 1 +492 1 +493 1 +494 1 +495 1 +496 1 +497 1 +498 1 +5 1 +51 1 +53 1 +54 1 +57 1 +58 1 +64 1 +65 1 +66 1 +67 1 +69 1 +70 1 +72 1 +74 1 +76 1 +77 1 +78 1 +8 1 +80 1 +82 1 +83 1 +84 1 +85 1 +86 1 +87 1 +9 1 +90 1 +92 1 +95 1 +96 1 +97 1 98 1 -97 1 -96 1 -95 1 -92 1 -90 1 -9 1 -87 1 -86 1 -85 1 -84 1 -83 1 -82 1 -80 1 -8 1 -78 1 -77 1 -76 1 -74 1 -72 1 -70 1 -69 1 -67 1 -66 1 -65 1 -64 1 -58 1 -57 1 -54 1 -53 1 -51 1 -5 1 -498 1 -497 1 -496 1 -495 1 -494 1 -493 1 -492 1 -491 1 -490 1 -489 1 -487 1 -485 1 -484 1 -483 1 -482 1 -481 1 -480 1 -479 1 -478 1 -477 1 -475 1 -472 1 -470 1 -47 1 -469 1 -468 1 -467 1 -466 1 -463 1 -462 1 -460 1 -459 1 -458 1 -457 1 -455 1 -454 1 -453 1 -452 1 -449 1 -448 1 -446 1 -444 1 -443 1 -44 1 -439 1 -438 1 -437 1 -436 1 -435 1 -432 1 -431 1 -430 1 -43 1 -429 1 -427 1 -424 1 -421 1 -42 1 -419 1 -418 1 -417 1 -414 1 -413 1 -411 1 -41 1 -409 1 -407 1 -406 1 -404 1 -403 1 -402 1 -401 1 -400 1 -4 1 -399 1 -397 1 -396 1 -395 1 -394 1 -393 1 -392 1 -389 1 -386 1 -384 1 -382 1 -379 1 -378 1 -377 1 -375 1 -374 1 -373 1 -37 1 -369 1 -368 1 -367 1 -366 1 -365 1 -364 1 -362 1 -360 1 -356 1 -353 1 -351 1 -35 1 -348 1 -345 1 -344 1 -342 1 -341 1 -34 1 -339 1 -338 1 -336 1 -335 1 -333 1 -332 1 -331 1 -33 1 -327 1 -325 1 -323 1 -322 1 -321 1 -318 1 -317 1 -316 1 -315 1 -311 1 -310 1 -309 1 -308 1 -307 1 -306 1 -305 1 -302 1 -30 1 -298 1 -296 1 -292 1 -291 1 -289 1 -288 1 -287 1 -286 1 -285 1 -284 1 -283 1 -282 1 -281 1 -280 1 -28 1 -278 1 -277 1 -275 1 -274 1 -273 1 -272 1 -27 1 -266 1 -265 1 -263 1 -262 1 -260 1 -26 1 -258 1 -257 1 -256 1 -255 1 -252 1 -249 1 -248 1 -247 1 -244 1 -242 1 -241 1 -24 1 -239 1 -238 1 -237 1 -235 1 -233 1 -230 1 -229 1 -228 1 -226 1 -224 1 -223 1 -222 1 -221 1 -219 1 -218 1 -217 1 -216 1 -214 1 -213 1 -209 1 -208 1 -207 1 -205 1 -203 1 -202 1 -201 1 -200 1 -20 1 -2 1 -199 1 -197 1 -196 1 -195 1 -194 1 -193 1 -192 1 -191 1 -190 1 -19 1 -189 1 -187 1 -186 1 -183 1 -181 1 -180 1 -18 1 -179 1 -178 1 -177 1 -176 1 -175 1 -174 1 -172 1 -170 1 -17 1 -169 1 -168 1 -167 1 -166 1 -165 1 -164 1 -163 1 -162 1 -160 1 -158 1 -157 1 -156 1 -155 1 -153 1 -152 1 -150 1 -15 1 -149 1 -146 1 -145 1 -143 1 -138 1 -137 1 -136 1 -134 1 -133 1 -131 1 -129 1 -128 1 -126 1 -125 1 -120 1 -12 1 -119 1 -118 1 -116 1 -114 1 -113 1 -111 1 -11 1 -105 1 -104 1 -103 1 -100 1 -10 1 -0 1 PREHOOK: query: -- same query with broadcast join -EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt +EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt,s2.key PREHOOK: type: QUERY POSTHOOK: query: -- same query with broadcast join -EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt +EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt,s2.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -890,14 +888,13 @@ outputColumnNames: _col0, _col1 Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: bigint) - sort order: + + key expressions: _col1 (type: bigint), _col0 (type: string) + sort order: ++ Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) Reducer 3 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint) + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -914,323 +911,323 @@ Processor Tree: ListSink -PREHOOK: query: SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt +PREHOOK: query: SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt,s2.key PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt +POSTHOOK: query: SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt,s2.key POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +0 1 +10 1 +100 1 +103 1 +104 1 +105 1 +11 1 +111 1 +113 1 +114 1 +116 1 +118 1 +119 1 +12 1 +120 1 +125 1 +126 1 +128 1 +129 1 +131 1 +133 1 +134 1 +136 1 +137 1 +138 1 +143 1 +145 1 +146 1 +149 1 +15 1 +150 1 +152 1 +153 1 +155 1 +156 1 +157 1 +158 1 +160 1 +162 1 +163 1 +164 1 +165 1 +166 1 +167 1 +168 1 +169 1 +17 1 +170 1 +172 1 +174 1 +175 1 +176 1 +177 1 +178 1 +179 1 +18 1 +180 1 +181 1 +183 1 +186 1 +187 1 +189 1 +19 1 +190 1 +191 1 +192 1 +193 1 +194 1 +195 1 +196 1 +197 1 +199 1 +2 1 +20 1 +200 1 +201 1 +202 1 +203 1 +205 1 +207 1 +208 1 +209 1 +213 1 +214 1 +216 1 +217 1 +218 1 +219 1 +221 1 +222 1 +223 1 +224 1 +226 1 +228 1 +229 1 +230 1 +233 1 +235 1 +237 1 +238 1 +239 1 +24 1 +241 1 +242 1 +244 1 +247 1 +248 1 +249 1 +252 1 +255 1 +256 1 +257 1 +258 1 +26 1 +260 1 +262 1 +263 1 +265 1 +266 1 +27 1 +272 1 +273 1 +274 1 +275 1 +277 1 +278 1 +28 1 +280 1 +281 1 +282 1 +283 1 +284 1 +285 1 +286 1 +287 1 +288 1 +289 1 +291 1 +292 1 +296 1 +298 1 +30 1 +302 1 +305 1 +306 1 +307 1 +308 1 +309 1 +310 1 +311 1 +315 1 +316 1 +317 1 +318 1 +321 1 +322 1 +323 1 +325 1 +327 1 +33 1 +331 1 +332 1 +333 1 +335 1 +336 1 +338 1 +339 1 +34 1 +341 1 +342 1 +344 1 +345 1 +348 1 +35 1 +351 1 +353 1 +356 1 +360 1 +362 1 +364 1 +365 1 +366 1 +367 1 +368 1 +369 1 +37 1 +373 1 +374 1 +375 1 +377 1 +378 1 +379 1 +382 1 +384 1 +386 1 +389 1 +392 1 +393 1 +394 1 +395 1 +396 1 +397 1 +399 1 +4 1 +400 1 +401 1 +402 1 +403 1 +404 1 +406 1 +407 1 +409 1 +41 1 +411 1 +413 1 +414 1 +417 1 +418 1 +419 1 +42 1 +421 1 +424 1 +427 1 +429 1 +43 1 +430 1 +431 1 +432 1 +435 1 +436 1 +437 1 +438 1 +439 1 +44 1 +443 1 +444 1 +446 1 +448 1 +449 1 +452 1 +453 1 +454 1 +455 1 +457 1 +458 1 +459 1 +460 1 +462 1 +463 1 +466 1 +467 1 +468 1 +469 1 +47 1 +470 1 +472 1 +475 1 +477 1 +478 1 +479 1 +480 1 +481 1 +482 1 +483 1 +484 1 +485 1 +487 1 +489 1 +490 1 +491 1 +492 1 +493 1 +494 1 +495 1 +496 1 +497 1 +498 1 +5 1 +51 1 +53 1 +54 1 +57 1 +58 1 +64 1 +65 1 +66 1 +67 1 +69 1 +70 1 +72 1 +74 1 +76 1 +77 1 +78 1 +8 1 +80 1 +82 1 +83 1 +84 1 +85 1 +86 1 +87 1 +9 1 +90 1 +92 1 +95 1 +96 1 +97 1 98 1 -97 1 -96 1 -95 1 -92 1 -90 1 -9 1 -87 1 -86 1 -85 1 -84 1 -83 1 -82 1 -80 1 -8 1 -78 1 -77 1 -76 1 -74 1 -72 1 -70 1 -69 1 -67 1 -66 1 -65 1 -64 1 -58 1 -57 1 -54 1 -53 1 -51 1 -5 1 -498 1 -497 1 -496 1 -495 1 -494 1 -493 1 -492 1 -491 1 -490 1 -489 1 -487 1 -485 1 -484 1 -483 1 -482 1 -481 1 -480 1 -479 1 -478 1 -477 1 -475 1 -472 1 -470 1 -47 1 -469 1 -468 1 -467 1 -466 1 -463 1 -462 1 -460 1 -459 1 -458 1 -457 1 -455 1 -454 1 -453 1 -452 1 -449 1 -448 1 -446 1 -444 1 -443 1 -44 1 -439 1 -438 1 -437 1 -436 1 -435 1 -432 1 -431 1 -430 1 -43 1 -429 1 -427 1 -424 1 -421 1 -42 1 -419 1 -418 1 -417 1 -414 1 -413 1 -411 1 -41 1 -409 1 -407 1 -406 1 -404 1 -403 1 -402 1 -401 1 -400 1 -4 1 -399 1 -397 1 -396 1 -395 1 -394 1 -393 1 -392 1 -389 1 -386 1 -384 1 -382 1 -379 1 -378 1 -377 1 -375 1 -374 1 -373 1 -37 1 -369 1 -368 1 -367 1 -366 1 -365 1 -364 1 -362 1 -360 1 -356 1 -353 1 -351 1 -35 1 -348 1 -345 1 -344 1 -342 1 -341 1 -34 1 -339 1 -338 1 -336 1 -335 1 -333 1 -332 1 -331 1 -33 1 -327 1 -325 1 -323 1 -322 1 -321 1 -318 1 -317 1 -316 1 -315 1 -311 1 -310 1 -309 1 -308 1 -307 1 -306 1 -305 1 -302 1 -30 1 -298 1 -296 1 -292 1 -291 1 -289 1 -288 1 -287 1 -286 1 -285 1 -284 1 -283 1 -282 1 -281 1 -280 1 -28 1 -278 1 -277 1 -275 1 -274 1 -273 1 -272 1 -27 1 -266 1 -265 1 -263 1 -262 1 -260 1 -26 1 -258 1 -257 1 -256 1 -255 1 -252 1 -249 1 -248 1 -247 1 -244 1 -242 1 -241 1 -24 1 -239 1 -238 1 -237 1 -235 1 -233 1 -230 1 -229 1 -228 1 -226 1 -224 1 -223 1 -222 1 -221 1 -219 1 -218 1 -217 1 -216 1 -214 1 -213 1 -209 1 -208 1 -207 1 -205 1 -203 1 -202 1 -201 1 -200 1 -20 1 -2 1 -199 1 -197 1 -196 1 -195 1 -194 1 -193 1 -192 1 -191 1 -190 1 -19 1 -189 1 -187 1 -186 1 -183 1 -181 1 -180 1 -18 1 -179 1 -178 1 -177 1 -176 1 -175 1 -174 1 -172 1 -170 1 -17 1 -169 1 -168 1 -167 1 -166 1 -165 1 -164 1 -163 1 -162 1 -160 1 -158 1 -157 1 -156 1 -155 1 -153 1 -152 1 -150 1 -15 1 -149 1 -146 1 -145 1 -143 1 -138 1 -137 1 -136 1 -134 1 -133 1 -131 1 -129 1 -128 1 -126 1 -125 1 -120 1 -12 1 -119 1 -118 1 -116 1 -114 1 -113 1 -111 1 -11 1 -105 1 -104 1 -103 1 -100 1 -10 1 -0 1 PREHOOK: query: -- query with multiple branches in the task dag EXPLAIN SELECT *