diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 25fba7f..ce726a8 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -93,6 +93,7 @@ import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooKeeper; +import org.junit.Assume; /** * QTestUtil. @@ -327,6 +328,12 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, Stri this.clusterType = clusterType; HadoopShims shims = ShimLoader.getHadoopShims(); + + // can run tez tests only on hadoop 2 + if (clusterType == MiniClusterType.tez) { + Assume.assumeTrue(ShimLoader.getMajorVersion().equals("0.23")); + } + if (clusterType != MiniClusterType.none) { dfs = shims.getMiniDfs(conf, 4, true, null); FileSystem fs = dfs.getFileSystem(); diff --git packaging/src/main/assembly/bin.xml packaging/src/main/assembly/bin.xml index 424d748..05095ee 100644 --- packaging/src/main/assembly/bin.xml +++ packaging/src/main/assembly/bin.xml @@ -40,6 +40,7 @@ true org.apache.hive.hcatalog:* + org.apache.tez:* diff --git ql/pom.xml ql/pom.xml index b54fa36..74e75f6 100644 --- ql/pom.xml +++ ql/pom.xml @@ -215,6 +215,21 @@ ${mockito-all.version} test + + org.apache.tez + tez-api + ${tez.version} + + + org.apache.tez + tez-runtime-library + ${tez.version} + + + org.apache.tez + tez-mapreduce + ${tez.version} + @@ -246,11 +261,6 @@ org.apache.tez - tez-api - ${tez.version} - - - org.apache.tez tez-dag ${tez.version} @@ -259,16 +269,6 @@ tez-common ${tez.version} - - org.apache.tez - tez-runtime-library - ${tez.version} - - - org.apache.tez - tez-mapreduce - ${tez.version} - org.apache.tez tez-runtime-internals diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java index 58fe62a..544fd82 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java @@ -80,12 +80,17 @@ JobConf conf; LocalResource appLr; Operator op; + Path path; + FileSystem fs; @SuppressWarnings("unchecked") @Before public void setUp() throws Exception { utils = mock(DagUtils.class); - when(utils.getTezDir(any(Path.class))).thenReturn(new Path("hdfs://localhost:9000/tez/")); + fs = mock(FileSystem.class); + path = mock(Path.class); + when(path.getFileSystem(any(Configuration.class))).thenReturn(fs); + when(utils.getTezDir(any(Path.class))).thenReturn(path); when(utils.createVertex(any(JobConf.class), any(BaseWork.class), any(Path.class), any(LocalResource.class), any(List.class), any(FileSystem.class), any(Context.class), anyBoolean())).thenAnswer(new Answer() { @@ -163,11 +168,13 @@ public void tearDown() throws Exception { utils = null; work = null; task = null; + path = null; + fs = null; } @Test public void testBuildDag() throws IllegalArgumentException, IOException, Exception { - DAG dag = task.build(conf, work, new Path("hdfs:///"), appLr, new Context(conf)); + DAG dag = task.build(conf, work, path, appLr, new Context(conf)); for (BaseWork w: work.getAllWork()) { Vertex v = dag.getVertex(w.getName()); assertNotNull(v); @@ -187,7 +194,7 @@ public void testBuildDag() throws IllegalArgumentException, IOException, Excepti @Test public void testEmptyWork() throws IllegalArgumentException, IOException, Exception { - DAG dag = task.build(conf, new TezWork(), new Path("hdfs:///"), appLr, new Context(conf)); + DAG dag = task.build(conf, new TezWork(), path, appLr, new Context(conf)); assertEquals(dag.getVertices().size(), 0); } @@ -195,7 +202,7 @@ public void testEmptyWork() throws IllegalArgumentException, IOException, Except public void testSubmit() throws LoginException, IllegalArgumentException, IOException, TezException, InterruptedException, URISyntaxException, HiveException { DAG dag = new DAG("test"); - task.submit(conf, dag, new Path("hdfs:///"), appLr, sessionState); + task.submit(conf, dag, path, appLr, sessionState); // validate close/reopen verify(sessionState, times(1)).open(any(String.class), any(HiveConf.class)); verify(sessionState, times(1)).close(eq(true)); diff --git ql/src/test/results/clientpositive/mrr.q.out ql/src/test/results/clientpositive/mrr.q.out index a694a0b..f7a5449 100644 --- ql/src/test/results/clientpositive/mrr.q.out +++ ql/src/test/results/clientpositive/mrr.q.out @@ -88,7 +88,6 @@ STAGE PLANS: Fetch Operator limit: -1 - PREHOOK: query: SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -536,7 +535,6 @@ STAGE PLANS: Fetch Operator limit: -1 - PREHOOK: query: SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -978,7 +976,6 @@ STAGE PLANS: Fetch Operator limit: -1 - PREHOOK: query: SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -1410,7 +1407,7 @@ STAGE PLANS: Map-reduce partition columns: expr: _col0 type: string - tag: 1 + tag: 2 value expressions: expr: _col0 type: string @@ -1479,7 +1476,13 @@ STAGE PLANS: Reduce Operator Tree: Extract Reduce Output Operator - sort order: + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string tag: 1 value expressions: expr: _col0 @@ -1549,7 +1552,13 @@ STAGE PLANS: Reduce Operator Tree: Extract Reduce Output Operator - sort order: + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string tag: 0 value expressions: expr: _col0 @@ -1560,56 +1569,27 @@ STAGE PLANS: Join Operator condition map: Inner Join 0 to 1 + Inner Join 0 to 2 condition expressions: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} {VALUE._col1} + 2 {VALUE._col0} {VALUE._col1} handleSkewJoin: false - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Filter Operator predicate: - expr: (_col0 = _col2) + expr: (_col1 > 1) type: boolean - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: 0 - value expressions: - expr: _col2 - type: string - expr: _col3 - type: bigint + Select Operator + expressions: expr: _col0 type: string expr: _col1 type: bigint - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} - 1 {VALUE._col0} {VALUE._col1} - handleSkewJoin: false - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Filter Operator - predicate: - expr: (_col3 > 1) - type: boolean - Select Operator - expressions: expr: _col2 type: string expr: _col3 type: bigint - expr: _col0 - type: string - expr: _col1 - type: bigint expr: _col4 type: string expr: _col5 @@ -1648,7 +1628,6 @@ STAGE PLANS: Fetch Operator limit: -1 - PREHOOK: query: SELECT * FROM (SELECT key, count(value) as cnt @@ -1875,24 +1854,6 @@ STAGE PLANS: value expressions: expr: _col1 type: bigint - Alias -> Map Operator Tree: - src - TableScan - alias: src - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: value - type: string Reduce Operator Tree: Group By Operator aggregations: @@ -1910,6 +1871,24 @@ STAGE PLANS: expr: _col1 type: bigint outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + Map-reduce partition columns: + expr: _col0 + type: string + tag: 0 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: bigint + Alias -> Map Operator Tree: + src + TableScan + alias: src Map Join Operator condition map: Inner Join 0 to 1 @@ -1921,7 +1900,7 @@ STAGE PLANS: 0 [Column[_col0]] 1 [Column[key]] outputColumnNames: _col0, _col1, _col2, _col3 - Position of Big Table: 0 + Position of Big Table: 1 Select Operator expressions: expr: _col0 @@ -1945,7 +1924,6 @@ STAGE PLANS: Fetch Operator limit: -1 - PREHOOK: query: SELECT * FROM (SELECT key, count(value) as cnt FROM src GROUP BY key) s1 @@ -1960,312 +1938,503 @@ FROM POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -0 3 0 val_0 -10 1 10 val_10 -100 2 100 val_100 -103 2 103 val_103 -104 2 104 val_104 -105 1 105 val_105 -11 1 11 val_11 -111 1 111 val_111 -113 2 113 val_113 -114 1 114 val_114 -116 1 116 val_116 -118 2 118 val_118 -119 3 119 val_119 -12 2 12 val_12 -120 2 120 val_120 -125 2 125 val_125 -126 1 126 val_126 +238 2 238 val_238 +86 1 86 val_86 +311 3 311 val_311 +27 1 27 val_27 +165 2 165 val_165 +409 3 409 val_409 +255 2 255 val_255 +278 2 278 val_278 +98 2 98 val_98 +484 1 484 val_484 +265 2 265 val_265 +193 3 193 val_193 +401 5 401 val_401 +150 1 150 val_150 +273 3 273 val_273 +224 2 224 val_224 +369 3 369 val_369 +66 1 66 val_66 128 3 128 val_128 -129 2 129 val_129 -131 1 131 val_131 -133 1 133 val_133 -134 2 134 val_134 -136 1 136 val_136 -137 2 137 val_137 -138 4 138 val_138 -143 1 143 val_143 -145 1 145 val_145 +213 2 213 val_213 146 2 146 val_146 -149 2 149 val_149 -15 2 15 val_15 -150 1 150 val_150 +406 4 406 val_406 +429 2 429 val_429 +374 1 374 val_374 152 2 152 val_152 +469 5 469 val_469 +145 1 145 val_145 +495 1 495 val_495 +37 2 37 val_37 +327 3 327 val_327 +281 2 281 val_281 +277 4 277 val_277 +209 2 209 val_209 +15 2 15 val_15 +82 1 82 val_82 +403 3 403 val_403 +166 1 166 val_166 +417 3 417 val_417 +430 3 430 val_430 +252 1 252 val_252 +292 1 292 val_292 +219 2 219 val_219 +287 1 287 val_287 153 1 153 val_153 -155 1 155 val_155 -156 1 156 val_156 -157 1 157 val_157 -158 1 158 val_158 -160 1 160 val_160 +193 3 193 val_193 +338 1 338 val_338 +446 1 446 val_446 +459 2 459 val_459 +394 1 394 val_394 +237 2 237 val_237 +482 1 482 val_482 +174 2 174 val_174 +413 2 413 val_413 +494 1 494 val_494 +207 2 207 val_207 +199 3 199 val_199 +466 3 466 val_466 +208 3 208 val_208 +174 2 174 val_174 +399 2 399 val_399 +396 3 396 val_396 +247 1 247 val_247 +417 3 417 val_417 +489 4 489 val_489 162 1 162 val_162 -163 1 163 val_163 -164 2 164 val_164 -165 2 165 val_165 -166 1 166 val_166 +377 1 377 val_377 +397 2 397 val_397 +309 2 309 val_309 +365 1 365 val_365 +266 1 266 val_266 +439 2 439 val_439 +342 2 342 val_342 +367 2 367 val_367 +325 2 325 val_325 167 3 167 val_167 -168 1 168 val_168 -169 4 169 val_169 -17 1 17 val_17 -170 1 170 val_170 -172 2 172 val_172 -174 2 174 val_174 -175 2 175 val_175 -176 2 176 val_176 -177 1 177 val_177 -178 1 178 val_178 -179 2 179 val_179 -18 2 18 val_18 -180 1 180 val_180 -181 1 181 val_181 -183 1 183 val_183 -186 1 186 val_186 -187 3 187 val_187 -189 1 189 val_189 -19 1 19 val_19 -190 1 190 val_190 -191 2 191 val_191 -192 1 192 val_192 -193 3 193 val_193 -194 1 194 val_194 195 2 195 val_195 -196 1 196 val_196 -197 2 197 val_197 -199 3 199 val_199 -2 1 2 val_2 -20 1 20 val_20 -200 2 200 val_200 -201 1 201 val_201 -202 1 202 val_202 +475 1 475 val_475 +17 1 17 val_17 +113 2 113 val_113 +155 1 155 val_155 203 2 203 val_203 +339 1 339 val_339 +0 3 0 val_0 +455 1 455 val_455 +128 3 128 val_128 +311 3 311 val_311 +316 3 316 val_316 +57 1 57 val_57 +302 1 302 val_302 205 2 205 val_205 -207 2 207 val_207 -208 3 208 val_208 -209 2 209 val_209 -213 2 213 val_213 -214 1 214 val_214 -216 2 216 val_216 -217 2 217 val_217 -218 1 218 val_218 -219 2 219 val_219 -221 2 221 val_221 -222 1 222 val_222 -223 2 223 val_223 -224 2 224 val_224 -226 1 226 val_226 -228 1 228 val_228 -229 2 229 val_229 -230 5 230 val_230 -233 2 233 val_233 -235 1 235 val_235 -237 2 237 val_237 -238 2 238 val_238 +149 2 149 val_149 +438 3 438 val_438 +345 1 345 val_345 +129 2 129 val_129 +170 1 170 val_170 +20 1 20 val_20 +489 4 489 val_489 +157 1 157 val_157 +378 1 378 val_378 +221 2 221 val_221 +92 1 92 val_92 +111 1 111 val_111 +47 1 47 val_47 +72 2 72 val_72 +4 1 4 val_4 +280 2 280 val_280 +35 3 35 val_35 +427 1 427 val_427 +277 4 277 val_277 +208 3 208 val_208 +356 1 356 val_356 +399 2 399 val_399 +169 4 169 val_169 +382 2 382 val_382 +498 3 498 val_498 +125 2 125 val_125 +386 1 386 val_386 +437 1 437 val_437 +469 5 469 val_469 +192 1 192 val_192 +286 1 286 val_286 +187 3 187 val_187 +176 2 176 val_176 +54 1 54 val_54 +459 2 459 val_459 +51 2 51 val_51 +138 4 138 val_138 +103 2 103 val_103 239 2 239 val_239 -24 2 24 val_24 +213 2 213 val_213 +216 2 216 val_216 +430 3 430 val_430 +278 2 278 val_278 +176 2 176 val_176 +289 1 289 val_289 +221 2 221 val_221 +65 1 65 val_65 +318 3 318 val_318 +332 1 332 val_332 +311 3 311 val_311 +275 1 275 val_275 +137 2 137 val_137 241 1 241 val_241 -242 2 242 val_242 -244 1 244 val_244 -247 1 247 val_247 -248 1 248 val_248 -249 1 249 val_249 -252 1 252 val_252 -255 2 255 val_255 -256 2 256 val_256 -257 1 257 val_257 -258 1 258 val_258 -26 2 26 val_26 +83 2 83 val_83 +333 2 333 val_333 +180 1 180 val_180 +284 1 284 val_284 +12 2 12 val_12 +230 5 230 val_230 +181 1 181 val_181 +67 2 67 val_67 260 1 260 val_260 -262 1 262 val_262 -263 1 263 val_263 -265 2 265 val_265 -266 1 266 val_266 -27 1 27 val_27 +404 2 404 val_404 +384 3 384 val_384 +489 4 489 val_489 +353 2 353 val_353 +373 1 373 val_373 272 2 272 val_272 -273 3 273 val_273 -274 1 274 val_274 -275 1 275 val_275 -277 4 277 val_277 -278 2 278 val_278 -28 1 28 val_28 -280 2 280 val_280 -281 2 281 val_281 -282 2 282 val_282 -283 1 283 val_283 -284 1 284 val_284 -285 1 285 val_285 -286 1 286 val_286 -287 1 287 val_287 -288 2 288 val_288 -289 1 289 val_289 -291 1 291 val_291 -292 1 292 val_292 -296 1 296 val_296 +138 4 138 val_138 +217 2 217 val_217 +84 2 84 val_84 +348 5 348 val_348 +466 3 466 val_466 +58 2 58 val_58 +8 1 8 val_8 +411 1 411 val_411 +230 5 230 val_230 +208 3 208 val_208 +348 5 348 val_348 +24 2 24 val_24 +463 2 463 val_463 +431 3 431 val_431 +179 2 179 val_179 +172 2 172 val_172 +42 2 42 val_42 +129 2 129 val_129 +158 1 158 val_158 +119 3 119 val_119 +496 1 496 val_496 +0 3 0 val_0 +322 2 322 val_322 +197 2 197 val_197 +468 4 468 val_468 +393 1 393 val_393 +454 3 454 val_454 +100 2 100 val_100 298 3 298 val_298 -30 1 30 val_30 -302 1 302 val_302 -305 1 305 val_305 -306 1 306 val_306 -307 2 307 val_307 +199 3 199 val_199 +191 2 191 val_191 +418 1 418 val_418 +96 1 96 val_96 +26 2 26 val_26 +165 2 165 val_165 +327 3 327 val_327 +230 5 230 val_230 +205 2 205 val_205 +120 2 120 val_120 +131 1 131 val_131 +51 2 51 val_51 +404 2 404 val_404 +43 1 43 val_43 +436 1 436 val_436 +156 1 156 val_156 +469 5 469 val_469 +468 4 468 val_468 308 1 308 val_308 -309 2 309 val_309 -310 1 310 val_310 -311 3 311 val_311 -315 1 315 val_315 -316 3 316 val_316 -317 2 317 val_317 +95 2 95 val_95 +196 1 196 val_196 +288 2 288 val_288 +481 1 481 val_481 +457 1 457 val_457 +98 2 98 val_98 +282 2 282 val_282 +197 2 197 val_197 +187 3 187 val_187 318 3 318 val_318 -321 2 321 val_321 -322 2 322 val_322 -323 1 323 val_323 -325 2 325 val_325 +318 3 318 val_318 +409 3 409 val_409 +470 1 470 val_470 +137 2 137 val_137 +369 3 369 val_369 +316 3 316 val_316 +169 4 169 val_169 +413 2 413 val_413 +85 1 85 val_85 +77 1 77 val_77 +0 3 0 val_0 +490 1 490 val_490 +87 1 87 val_87 +364 1 364 val_364 +179 2 179 val_179 +118 2 118 val_118 +134 2 134 val_134 +395 2 395 val_395 +282 2 282 val_282 +138 4 138 val_138 +238 2 238 val_238 +419 1 419 val_419 +15 2 15 val_15 +118 2 118 val_118 +72 2 72 val_72 +90 3 90 val_90 +307 2 307 val_307 +19 1 19 val_19 +435 1 435 val_435 +10 1 10 val_10 +277 4 277 val_277 +273 3 273 val_273 +306 1 306 val_306 +224 2 224 val_224 +309 2 309 val_309 +389 1 389 val_389 327 3 327 val_327 -33 1 33 val_33 +242 2 242 val_242 +369 3 369 val_369 +392 1 392 val_392 +272 2 272 val_272 331 2 331 val_331 -332 1 332 val_332 -333 2 333 val_333 -335 1 335 val_335 +401 5 401 val_401 +242 2 242 val_242 +452 1 452 val_452 +177 1 177 val_177 +226 1 226 val_226 +5 3 5 val_5 +497 1 497 val_497 +402 1 402 val_402 +396 3 396 val_396 +317 2 317 val_317 +395 2 395 val_395 +58 2 58 val_58 +35 3 35 val_35 336 1 336 val_336 -338 1 338 val_338 -339 1 339 val_339 +95 2 95 val_95 +11 1 11 val_11 +168 1 168 val_168 34 1 34 val_34 -341 1 341 val_341 +229 2 229 val_229 +233 2 233 val_233 +143 1 143 val_143 +472 1 472 val_472 +322 2 322 val_322 +498 3 498 val_498 +160 1 160 val_160 +195 2 195 val_195 +42 2 42 val_42 +321 2 321 val_321 +430 3 430 val_430 +119 3 119 val_119 +489 4 489 val_489 +458 2 458 val_458 +78 1 78 val_78 +76 2 76 val_76 +41 1 41 val_41 +223 2 223 val_223 +492 2 492 val_492 +149 2 149 val_149 +449 1 449 val_449 +218 1 218 val_218 +228 1 228 val_228 +138 4 138 val_138 +453 1 453 val_453 +30 1 30 val_30 +209 2 209 val_209 +64 1 64 val_64 +468 4 468 val_468 +76 2 76 val_76 +74 1 74 val_74 342 2 342 val_342 +69 1 69 val_69 +230 5 230 val_230 +33 1 33 val_33 +368 1 368 val_368 +103 2 103 val_103 +296 1 296 val_296 +113 2 113 val_113 +216 2 216 val_216 +367 2 367 val_367 344 2 344 val_344 -345 1 345 val_345 -348 5 348 val_348 +167 3 167 val_167 +274 1 274 val_274 +219 2 219 val_219 +239 2 239 val_239 +485 1 485 val_485 +116 1 116 val_116 +223 2 223 val_223 +256 2 256 val_256 +263 1 263 val_263 +70 3 70 val_70 +487 1 487 val_487 +480 3 480 val_480 +401 5 401 val_401 +288 2 288 val_288 +191 2 191 val_191 +5 3 5 val_5 +244 1 244 val_244 +438 3 438 val_438 +128 3 128 val_128 +467 1 467 val_467 +432 1 432 val_432 +202 1 202 val_202 +316 3 316 val_316 +229 2 229 val_229 +469 5 469 val_469 +463 2 463 val_463 +280 2 280 val_280 +2 1 2 val_2 35 3 35 val_35 -351 1 351 val_351 -353 2 353 val_353 -356 1 356 val_356 -360 1 360 val_360 -362 1 362 val_362 -364 1 364 val_364 -365 1 365 val_365 +283 1 283 val_283 +331 2 331 val_331 +235 1 235 val_235 +80 1 80 val_80 +44 1 44 val_44 +193 3 193 val_193 +321 2 321 val_321 +335 1 335 val_335 +104 2 104 val_104 +466 3 466 val_466 366 1 366 val_366 -367 2 367 val_367 -368 1 368 val_368 -369 3 369 val_369 -37 2 37 val_37 -373 1 373 val_373 -374 1 374 val_374 -375 1 375 val_375 -377 1 377 val_377 -378 1 378 val_378 -379 1 379 val_379 -382 2 382 val_382 -384 3 384 val_384 -386 1 386 val_386 -389 1 389 val_389 -392 1 392 val_392 -393 1 393 val_393 -394 1 394 val_394 -395 2 395 val_395 -396 3 396 val_396 -397 2 397 val_397 -399 2 399 val_399 -4 1 4 val_4 -400 1 400 val_400 -401 5 401 val_401 -402 1 402 val_402 +175 2 175 val_175 403 3 403 val_403 -404 2 404 val_404 +483 1 483 val_483 +53 1 53 val_53 +105 1 105 val_105 +257 1 257 val_257 406 4 406 val_406 -407 1 407 val_407 409 3 409 val_409 -41 1 41 val_41 -411 1 411 val_411 -413 2 413 val_413 -414 2 414 val_414 -417 3 417 val_417 -418 1 418 val_418 -419 1 419 val_419 -42 2 42 val_42 -421 1 421 val_421 +190 1 190 val_190 +406 4 406 val_406 +401 5 401 val_401 +114 1 114 val_114 +258 1 258 val_258 +90 3 90 val_90 +203 2 203 val_203 +262 1 262 val_262 +348 5 348 val_348 424 2 424 val_424 -427 1 427 val_427 -429 2 429 val_429 -43 1 43 val_43 -430 3 430 val_430 +12 2 12 val_12 +396 3 396 val_396 +201 1 201 val_201 +217 2 217 val_217 +164 2 164 val_164 431 3 431 val_431 -432 1 432 val_432 -435 1 435 val_435 -436 1 436 val_436 -437 1 437 val_437 +454 3 454 val_454 +478 2 478 val_478 +298 3 298 val_298 +125 2 125 val_125 +431 3 431 val_431 +164 2 164 val_164 +424 2 424 val_424 +187 3 187 val_187 +382 2 382 val_382 +5 3 5 val_5 +70 3 70 val_70 +397 2 397 val_397 +480 3 480 val_480 +291 1 291 val_291 +24 2 24 val_24 +351 1 351 val_351 +255 2 255 val_255 +104 2 104 val_104 +70 3 70 val_70 +163 1 163 val_163 438 3 438 val_438 +119 3 119 val_119 +414 2 414 val_414 +200 2 200 val_200 +491 1 491 val_491 +237 2 237 val_237 439 2 439 val_439 -44 1 44 val_44 -443 1 443 val_443 +360 1 360 val_360 +248 1 248 val_248 +479 1 479 val_479 +305 1 305 val_305 +417 3 417 val_417 +199 3 199 val_199 444 1 444 val_444 -446 1 446 val_446 -448 1 448 val_448 -449 1 449 val_449 -452 1 452 val_452 -453 1 453 val_453 -454 3 454 val_454 -455 1 455 val_455 -457 1 457 val_457 -458 2 458 val_458 -459 2 459 val_459 -460 1 460 val_460 -462 2 462 val_462 -463 2 463 val_463 -466 3 466 val_466 -467 1 467 val_467 -468 4 468 val_468 -469 5 469 val_469 -47 1 47 val_47 -470 1 470 val_470 -472 1 472 val_472 -475 1 475 val_475 -477 1 477 val_477 +120 2 120 val_120 +429 2 429 val_429 +169 4 169 val_169 +443 1 443 val_443 +323 1 323 val_323 +325 2 325 val_325 +277 4 277 val_277 +230 5 230 val_230 478 2 478 val_478 -479 1 479 val_479 +178 1 178 val_178 +468 4 468 val_468 +310 1 310 val_310 +317 2 317 val_317 +333 2 333 val_333 +493 1 493 val_493 +460 1 460 val_460 +207 2 207 val_207 +249 1 249 val_249 +265 2 265 val_265 480 3 480 val_480 -481 1 481 val_481 -482 1 482 val_482 -483 1 483 val_483 -484 1 484 val_484 -485 1 485 val_485 -487 1 487 val_487 -489 4 489 val_489 -490 1 490 val_490 -491 1 491 val_491 +83 2 83 val_83 +136 1 136 val_136 +353 2 353 val_353 +172 2 172 val_172 +214 1 214 val_214 +462 2 462 val_462 +233 2 233 val_233 +406 4 406 val_406 +133 1 133 val_133 +175 2 175 val_175 +189 1 189 val_189 +454 3 454 val_454 +375 1 375 val_375 +401 5 401 val_401 +421 1 421 val_421 +407 1 407 val_407 +384 3 384 val_384 +256 2 256 val_256 +26 2 26 val_26 +134 2 134 val_134 +67 2 67 val_67 +384 3 384 val_384 +379 1 379 val_379 +18 2 18 val_18 +462 2 462 val_462 492 2 492 val_492 -493 1 493 val_493 -494 1 494 val_494 -495 1 495 val_495 -496 1 496 val_496 -497 1 497 val_497 +100 2 100 val_100 +298 3 298 val_298 +9 1 9 val_9 +341 1 341 val_341 498 3 498 val_498 -5 3 5 val_5 -51 2 51 val_51 -53 1 53 val_53 -54 1 54 val_54 -57 1 57 val_57 -58 2 58 val_58 -64 1 64 val_64 -65 1 65 val_65 -66 1 66 val_66 -67 2 67 val_67 -69 1 69 val_69 -70 3 70 val_70 -72 2 72 val_72 -74 1 74 val_74 -76 2 76 val_76 -77 1 77 val_77 -78 1 78 val_78 -8 1 8 val_8 -80 1 80 val_80 -82 1 82 val_82 -83 2 83 val_83 +146 2 146 val_146 +458 2 458 val_458 +362 1 362 val_362 +186 1 186 val_186 +285 1 285 val_285 +348 5 348 val_348 +167 3 167 val_167 +18 2 18 val_18 +273 3 273 val_273 +183 1 183 val_183 +281 2 281 val_281 +344 2 344 val_344 +97 2 97 val_97 +469 5 469 val_469 +315 1 315 val_315 84 2 84 val_84 -85 1 85 val_85 -86 1 86 val_86 -87 1 87 val_87 -9 1 9 val_9 +28 1 28 val_28 +37 2 37 val_37 +448 1 448 val_448 +152 2 152 val_152 +348 5 348 val_348 +307 2 307 val_307 +194 1 194 val_194 +414 2 414 val_414 +477 1 477 val_477 +222 1 222 val_222 +126 1 126 val_126 90 3 90 val_90 -92 1 92 val_92 -95 2 95 val_95 -96 1 96 val_96 +169 4 169 val_169 +403 3 403 val_403 +400 1 400 val_400 +200 2 200 val_200 97 2 97 val_97 -98 2 98 val_98 diff --git ql/src/test/results/clientpositive/tez_dml.q.out ql/src/test/results/clientpositive/tez_dml.q.out index e5a140e..0ada1e2 100644 --- ql/src/test/results/clientpositive/tez_dml.q.out +++ ql/src/test/results/clientpositive/tez_dml.q.out @@ -11,9 +11,10 @@ STAGE DEPENDENCIES: Stage-1 is a root stage Stage-8 depends on stages: Stage-1 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 - Stage-9 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-5, Stage-4, Stage-7 + Stage-9 depends on stages: Stage-2, Stage-0 Stage-3 depends on stages: Stage-9 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 Stage-4 Stage-6 Stage-7 depends on stages: Stage-6 @@ -99,11 +100,8 @@ STAGE PLANS: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-0 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Stage: Stage-2 + Dependency Collection Stage: Stage-9 Create Table Operator: @@ -119,8 +117,14 @@ STAGE PLANS: Stage: Stage-3 Stats-Aggr Operator + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + Stage: Stage-4 - Map Reduce + Tez Alias -> Map Operator Tree: #### A masked pattern was here #### TableScan @@ -134,7 +138,7 @@ STAGE PLANS: name: default.tmp_src Stage: Stage-6 - Map Reduce + Tez Alias -> Map Operator Tree: #### A masked pattern was here #### TableScan @@ -153,7 +157,6 @@ STAGE PLANS: hdfs directory: true #### A masked pattern was here #### - PREHOOK: query: CREATE TABLE tmp_src AS SELECT * FROM (SELECT value, count(value) AS cnt FROM src GROUP BY value) f1 ORDER BY cnt PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src @@ -496,7 +499,8 @@ STAGE DEPENDENCIES: Stage-1 is a root stage Stage-8 depends on stages: Stage-1 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 + Stage-2 depends on stages: Stage-5, Stage-4, Stage-7 + Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 Stage-4 Stage-6 @@ -534,6 +538,9 @@ STAGE PLANS: hdfs directory: true #### A masked pattern was here #### + Stage: Stage-2 + Dependency Collection + Stage: Stage-0 Move Operator tables: @@ -550,7 +557,7 @@ STAGE PLANS: Stats-Aggr Operator Stage: Stage-4 - Map Reduce + Tez Alias -> Map Operator Tree: #### A masked pattern was here #### TableScan @@ -564,7 +571,7 @@ STAGE PLANS: name: default.tmp_src_part Stage: Stage-6 - Map Reduce + Tez Alias -> Map Operator Tree: #### A masked pattern was here #### TableScan @@ -583,7 +590,6 @@ STAGE PLANS: hdfs directory: true #### A masked pattern was here #### - PREHOOK: query: INSERT INTO TABLE tmp_src_part PARTITION (d) SELECT * FROM tmp_src PREHOOK: type: QUERY PREHOOK: Input: default@tmp_src @@ -977,15 +983,16 @@ STAGE DEPENDENCIES: Stage-2 is a root stage Stage-9 depends on stages: Stage-2 , consists of Stage-6, Stage-5, Stage-7 Stage-6 - Stage-0 depends on stages: Stage-6, Stage-5, Stage-8 + Stage-3 depends on stages: Stage-6, Stage-5, Stage-8, Stage-12, Stage-11, Stage-14 + Stage-0 depends on stages: Stage-3 Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-10 depends on stages: Stage-1 Stage-5 Stage-7 Stage-8 depends on stages: Stage-7 Stage-15 depends on stages: Stage-2 , consists of Stage-12, Stage-11, Stage-13 Stage-12 - Stage-1 depends on stages: Stage-12, Stage-11, Stage-14 - Stage-10 depends on stages: Stage-1 Stage-11 Stage-13 Stage-14 depends on stages: Stage-13 @@ -1045,6 +1052,9 @@ STAGE PLANS: hdfs directory: true #### A masked pattern was here #### + Stage: Stage-3 + Dependency Collection + Stage: Stage-0 Move Operator tables: @@ -1058,8 +1068,21 @@ STAGE PLANS: Stage: Stage-4 Stats-Aggr Operator + Stage: Stage-1 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.odd + + Stage: Stage-10 + Stats-Aggr Operator + Stage: Stage-5 - Map Reduce + Tez Alias -> Map Operator Tree: #### A masked pattern was here #### TableScan @@ -1073,7 +1096,7 @@ STAGE PLANS: name: default.even Stage: Stage-7 - Map Reduce + Tez Alias -> Map Operator Tree: #### A masked pattern was here #### TableScan @@ -1101,21 +1124,8 @@ STAGE PLANS: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-1 - Move Operator - tables: - replace: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.odd - - Stage: Stage-10 - Stats-Aggr Operator - Stage: Stage-11 - Map Reduce + Tez Alias -> Map Operator Tree: #### A masked pattern was here #### TableScan @@ -1129,7 +1139,7 @@ STAGE PLANS: name: default.odd Stage: Stage-13 - Map Reduce + Tez Alias -> Map Operator Tree: #### A masked pattern was here #### TableScan @@ -1148,7 +1158,6 @@ STAGE PLANS: hdfs directory: true #### A masked pattern was here #### - PREHOOK: query: FROM src INSERT INTO TABLE even SELECT key, value WHERE key % 2 = 0 INSERT INTO TABLE odd SELECT key, value WHERE key % 2 = 1 diff --git ql/src/test/results/clientpositive/tez_join_tests.q.out ql/src/test/results/clientpositive/tez_join_tests.q.out index 358f861..1dda624 100644 --- ql/src/test/results/clientpositive/tez_join_tests.q.out +++ ql/src/test/results/clientpositive/tez_join_tests.q.out @@ -155,7 +155,6 @@ STAGE PLANS: Fetch Operator limit: -1 - PREHOOK: query: select * from (select b.key, b.value from src1 a left outer join src b on (a.key = b.key) order by b.key) x right outer join src c on (x.value = c.value) order by x.key PREHOOK: type: QUERY PREHOOK: Input: default@src diff --git ql/src/test/results/clientpositive/tez_joins_explain.q.out ql/src/test/results/clientpositive/tez_joins_explain.q.out index b68da6e..18e1ce6 100644 --- ql/src/test/results/clientpositive/tez_joins_explain.q.out +++ ql/src/test/results/clientpositive/tez_joins_explain.q.out @@ -155,7 +155,6 @@ STAGE PLANS: Fetch Operator limit: -1 - PREHOOK: query: select * from (select b.key, b.value from src1 a left outer join src b on (a.key = b.key) order by b.key) x right outer join src c on (x.value = c.value) order by x.key PREHOOK: type: QUERY PREHOOK: Input: default@src