diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/PrintCompletedTasksHook.java itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/PrintCompletedTasksHook.java new file mode 100644 index 0000000..dae49e1 --- /dev/null +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/PrintCompletedTasksHook.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.hooks; + +import org.apache.hadoop.hive.ql.exec.TaskRunner; +import org.apache.hadoop.hive.ql.session.SessionState; + +/** + * List all tasks completed. + * + * This is used to check tasks executed at runtime for queries containing conditional tasks + */ +public class PrintCompletedTasksHook implements ExecuteWithHookContext { + + @Override + public void run(HookContext hookContext) { + if (hookContext.getHookType() == HookContext.HookType.POST_EXEC_HOOK) { + SessionState.LogHelper console = SessionState.getConsole(); + if (console != null && !hookContext.getQueryPlan().isExplain()) { + for (TaskRunner runner : hookContext.getCompleteTaskList()) { + console.printError("RUN: " + runner.getTask()); + } + } + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java index 1e47413..7e60bd1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java +++ ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java @@ -38,6 +38,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.exec.ConditionalTask; +import org.apache.hadoop.hive.ql.exec.ExplainTask; import org.apache.hadoop.hive.ql.exec.FetchTask; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.Task; @@ -392,9 +393,6 @@ private void extractCounters() throws IOException { done.add(task.getId() + "_MAP"); } if (mrTask.hasReduce()) { - Collection> reducerTopOps = - new ArrayList>(); - reducerTopOps.add(mrTask.getWork().getReduceWork().getReducer()); if (mrTask.reduceStarted()) { started.add(task.getId() + "_REDUCE"); } @@ -588,6 +586,10 @@ public String getJSONQuery(org.apache.hadoop.hive.ql.plan.api.Query query) { return sb.toString(); } + public boolean isExplain() { + return rootTasks.size() == 1 && rootTasks.get(0) instanceof ExplainTask; + } + @Override public String toString() { try { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java index 5ac6338..f2fabad 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java @@ -488,9 +488,8 @@ public static boolean cannotConvert(long aliasKnownSize, // create map join task and set big table as i MapRedTask newTask = convertTaskToMapJoinTask(newWork, pos); - MapWork mapWork = newTask.getWork().getMapWork(); Operator startOp = joinOp.getParentOperators().get(pos); - Set aliases = GenMapRedUtils.findAliases(mapWork, startOp); + Set aliases = GenMapRedUtils.findAliases(currWork, startOp); long aliasKnownSize = Utilities.sumOf(aliasToSize, aliases); if (cannotConvert(aliasKnownSize, aliasTotalKnownInputSize, ThresholdOfSmallTblSizeSum)) { diff --git ql/src/test/queries/clientpositive/auto_join_without_localtask.q ql/src/test/queries/clientpositive/auto_join_without_localtask.q index bb7edc9..f23e227 100644 --- ql/src/test/queries/clientpositive/auto_join_without_localtask.q +++ ql/src/test/queries/clientpositive/auto_join_without_localtask.q @@ -1,3 +1,4 @@ +set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecutePrinter,org.apache.hadoop.hive.ql.hooks.PrintCompletedTasksHook; set hive.auto.convert.join=true; set hive.auto.convert.join.use.nonstaged=true; diff --git ql/src/test/queries/clientpositive/multiMapJoin1.q ql/src/test/queries/clientpositive/multiMapJoin1.q index 9a0a792..455f550 100644 --- ql/src/test/queries/clientpositive/multiMapJoin1.q +++ ql/src/test/queries/clientpositive/multiMapJoin1.q @@ -1,3 +1,5 @@ +set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecutePrinter,org.apache.hadoop.hive.ql.hooks.PrintCompletedTasksHook; + create table smallTbl1(key string, value string); insert overwrite table smallTbl1 select * from src where key < 10; diff --git ql/src/test/queries/clientpositive/multiMapJoin2.q ql/src/test/queries/clientpositive/multiMapJoin2.q index 1ebd3d1..141db4d 100644 --- ql/src/test/queries/clientpositive/multiMapJoin2.q +++ ql/src/test/queries/clientpositive/multiMapJoin2.q @@ -1,3 +1,4 @@ +set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecutePrinter,org.apache.hadoop.hive.ql.hooks.PrintCompletedTasksHook; set hive.auto.convert.join=true; set hive.auto.convert.join.noconditionaltask=true; set hive.auto.convert.join.noconditionaltask.size=6000; diff --git ql/src/test/queries/clientpositive/subquery_multiinsert.q ql/src/test/queries/clientpositive/subquery_multiinsert.q index f65696c..ed36d9e 100644 --- ql/src/test/queries/clientpositive/subquery_multiinsert.q +++ ql/src/test/queries/clientpositive/subquery_multiinsert.q @@ -1,4 +1,6 @@ -CREATE TABLE src_4( +set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecutePrinter,org.apache.hadoop.hive.ql.hooks.PrintCompletedTasksHook; + +CREATE TABLE src_4( key STRING, value STRING ) diff --git ql/src/test/results/clientpositive/auto_join_without_localtask.q.out ql/src/test/results/clientpositive/auto_join_without_localtask.q.out index 3053d97..0c578f8 100644 --- ql/src/test/results/clientpositive/auto_join_without_localtask.q.out +++ ql/src/test/results/clientpositive/auto_join_without_localtask.q.out @@ -145,6 +145,8 @@ POSTHOOK: query: select a.* from src a join src b on a.key=b.key limit 40 POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +RUN: Stage-5:CONDITIONAL +RUN: Stage-3:MAPRED 238 val_238 238 val_238 86 val_86 @@ -435,6 +437,10 @@ POSTHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +RUN: Stage-10:CONDITIONAL +RUN: Stage-8:MAPRED +RUN: Stage-7:CONDITIONAL +RUN: Stage-5:MAPRED 238 val_238 238 val_238 238 val_238 @@ -761,6 +767,11 @@ POSTHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +RUN: Stage-10:CONDITIONAL +RUN: Stage-14:MAPREDLOCAL +RUN: Stage-9:MAPRED +RUN: Stage-7:CONDITIONAL +RUN: Stage-5:MAPRED 238 val_238 238 val_238 238 val_238 @@ -823,7 +834,7 @@ Obtaining error information Task failed! Task ID: - Stage-6 + Stage-5 Logs: @@ -835,6 +846,12 @@ select a.* from src a join src b on a.key=b.key join src c on a.value=c.value wh POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +RUN: Stage-10:CONDITIONAL +RUN: Stage-14:MAPREDLOCAL +RUN: Stage-1:MAPRED +RUN: Stage-7:CONDITIONAL +RUN: Stage-5:MAPRED +RUN: Stage-2:MAPRED 103 val_103 103 val_103 103 val_103 diff --git ql/src/test/results/clientpositive/multiMapJoin1.q.out ql/src/test/results/clientpositive/multiMapJoin1.q.out index 039a7ee..a73d47f 100644 --- ql/src/test/results/clientpositive/multiMapJoin1.q.out +++ ql/src/test/results/clientpositive/multiMapJoin1.q.out @@ -5,6 +5,7 @@ POSTHOOK: query: create table smallTbl1(key string, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@smallTbl1 +RUN: Stage-0:DDL PREHOOK: query: insert overwrite table smallTbl1 select * from src where key < 10 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -15,6 +16,11 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@smalltbl1 POSTHOOK: Lineage: smalltbl1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-1:MAPRED +RUN: Stage-7:CONDITIONAL +RUN: Stage-4:MOVE +RUN: Stage-0:MOVE +RUN: Stage-2:STATS PREHOOK: query: create table smallTbl2(key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -24,6 +30,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@smallTbl2 POSTHOOK: Lineage: smalltbl1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-0:DDL PREHOOK: query: insert overwrite table smallTbl2 select * from src where key < 10 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -36,6 +43,11 @@ POSTHOOK: Lineage: smalltbl1.key SIMPLE [(src)src.FieldSchema(name:key, type:str POSTHOOK: Lineage: smalltbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-1:MAPRED +RUN: Stage-7:CONDITIONAL +RUN: Stage-4:MOVE +RUN: Stage-0:MOVE +RUN: Stage-2:STATS PREHOOK: query: create table smallTbl3(key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -47,6 +59,7 @@ POSTHOOK: Lineage: smalltbl1.key SIMPLE [(src)src.FieldSchema(name:key, type:str POSTHOOK: Lineage: smalltbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-0:DDL PREHOOK: query: insert overwrite table smallTbl3 select * from src where key < 10 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -61,6 +74,11 @@ POSTHOOK: Lineage: smalltbl2.key SIMPLE [(src)src.FieldSchema(name:key, type:str POSTHOOK: Lineage: smalltbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl3.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-1:MAPRED +RUN: Stage-7:CONDITIONAL +RUN: Stage-4:MOVE +RUN: Stage-0:MOVE +RUN: Stage-2:STATS PREHOOK: query: create table smallTbl4(key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -74,6 +92,7 @@ POSTHOOK: Lineage: smalltbl2.key SIMPLE [(src)src.FieldSchema(name:key, type:str POSTHOOK: Lineage: smalltbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl3.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-0:DDL PREHOOK: query: insert overwrite table smallTbl4 select * from src where key < 10 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -90,6 +109,11 @@ POSTHOOK: Lineage: smalltbl3.key SIMPLE [(src)src.FieldSchema(name:key, type:str POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-1:MAPRED +RUN: Stage-7:CONDITIONAL +RUN: Stage-4:MOVE +RUN: Stage-0:MOVE +RUN: Stage-2:STATS PREHOOK: query: create table bigTbl(key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -105,6 +129,7 @@ POSTHOOK: Lineage: smalltbl3.key SIMPLE [(src)src.FieldSchema(name:key, type:str POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-0:DDL PREHOOK: query: insert overwrite table bigTbl select * from ( @@ -167,6 +192,11 @@ POSTHOOK: Lineage: smalltbl3.key SIMPLE [(src)src.FieldSchema(name:key, type:str POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-1:MAPRED +RUN: Stage-7:CONDITIONAL +RUN: Stage-4:MOVE +RUN: Stage-0:MOVE +RUN: Stage-2:STATS PREHOOK: query: explain select count(*) FROM (select bigTbl.key as key, bigTbl.value as value1, @@ -316,6 +346,7 @@ POSTHOOK: Lineage: smalltbl3.key SIMPLE [(src)src.FieldSchema(name:key, type:str POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-2:MAPRED 580 PREHOOK: query: -- Now run a query with two-way join, which should be converted into a -- map-join followed by groupby - two MR jobs overall @@ -470,6 +501,7 @@ POSTHOOK: Lineage: smalltbl3.key SIMPLE [(src)src.FieldSchema(name:key, type:str POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-2:MAPRED 580 PREHOOK: query: -- Now run a query with two-way join, which should first be converted into a -- map-join followed by groupby and then finally into a single MR job. @@ -637,6 +669,7 @@ POSTHOOK: Lineage: smalltbl3.key SIMPLE [(src)src.FieldSchema(name:key, type:str POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-2:MAPRED 270 10 10 @@ -661,6 +694,7 @@ POSTHOOK: Lineage: smalltbl3.key SIMPLE [(src)src.FieldSchema(name:key, type:str POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-0:DDL PREHOOK: query: create table bigTbl(key1 string, key2 string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -678,6 +712,7 @@ POSTHOOK: Lineage: smalltbl3.key SIMPLE [(src)src.FieldSchema(name:key, type:str POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-0:DDL PREHOOK: query: insert overwrite table bigTbl select * from ( @@ -743,6 +778,11 @@ POSTHOOK: Lineage: smalltbl3.key SIMPLE [(src)src.FieldSchema(name:key, type:str POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-1:MAPRED +RUN: Stage-7:CONDITIONAL +RUN: Stage-4:MOVE +RUN: Stage-0:MOVE +RUN: Stage-2:STATS PREHOOK: query: -- First disable noconditionaltask EXPLAIN SELECT SUM(HASH(join3.key1)), @@ -1421,6 +1461,15 @@ POSTHOOK: Lineage: smalltbl3.key SIMPLE [(src)src.FieldSchema(name:key, type:str POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-20:CONDITIONAL +RUN: Stage-18:MAPRED +RUN: Stage-17:CONDITIONAL +RUN: Stage-15:MAPRED +RUN: Stage-14:CONDITIONAL +RUN: Stage-12:MAPRED +RUN: Stage-11:CONDITIONAL +RUN: Stage-9:MAPRED +RUN: Stage-4:MAPRED 247580 247580 247580 247580 247580 247580 548662743780 548662743780 PREHOOK: query: -- Enable noconditionaltask and set the size of hive.auto.convert.join.noconditionaltask.size -- to 10000, which is large enough to fit all four small tables (smallTbl1 to smallTbl4). @@ -1730,6 +1779,7 @@ POSTHOOK: Lineage: smalltbl3.key SIMPLE [(src)src.FieldSchema(name:key, type:str POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-4:MAPRED 247580 247580 247580 247580 247580 247580 548662743780 548662743780 PREHOOK: query: -- Enable noconditionaltask and set the size of hive.auto.convert.join.noconditionaltask.size -- to 200, which is large enough to fit two small tables. We will have two jobs to evaluate this @@ -2057,6 +2107,8 @@ POSTHOOK: Lineage: smalltbl3.key SIMPLE [(src)src.FieldSchema(name:key, type:str POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-11:MAPRED +RUN: Stage-4:MAPRED 247580 247580 247580 247580 247580 247580 548662743780 548662743780 PREHOOK: query: -- Enable noconditionaltask and but set the size of hive.auto.convert.join.noconditionaltask.size -- to 0. The plan will be the same as the one with a disabled nonconditionaltask. @@ -2738,4 +2790,13 @@ POSTHOOK: Lineage: smalltbl3.key SIMPLE [(src)src.FieldSchema(name:key, type:str POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-20:CONDITIONAL +RUN: Stage-18:MAPRED +RUN: Stage-17:CONDITIONAL +RUN: Stage-15:MAPRED +RUN: Stage-14:CONDITIONAL +RUN: Stage-12:MAPRED +RUN: Stage-11:CONDITIONAL +RUN: Stage-9:MAPRED +RUN: Stage-4:MAPRED 247580 247580 247580 247580 247580 247580 548662743780 548662743780 diff --git ql/src/test/results/clientpositive/multiMapJoin2.q.out ql/src/test/results/clientpositive/multiMapJoin2.q.out index 33fb6dd..8c9b489 100644 --- ql/src/test/results/clientpositive/multiMapJoin2.q.out +++ ql/src/test/results/clientpositive/multiMapJoin2.q.out @@ -131,6 +131,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### +RUN: Stage-2:MAPRED 128 128 128 @@ -366,6 +367,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### +RUN: Stage-7:MAPRED +RUN: Stage-2:MAPRED 128 128 128 @@ -593,6 +596,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### +RUN: Stage-4:MAPRED +RUN: Stage-2:MAPRED 128 128 @@ -1015,6 +1020,12 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### +RUN: Stage-2:MAPRED +RUN: Stage-8:MAPRED +RUN: Stage-12:CONDITIONAL +RUN: Stage-11:MAPRED +RUN: Stage-4:MAPRED +RUN: Stage-5:MAPRED 128 1 146 1 150 1 @@ -1283,6 +1294,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### +RUN: Stage-2:MAPRED +RUN: Stage-3:MAPRED 128 1 146 1 150 1 @@ -1645,6 +1658,12 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### +RUN: Stage-7:MAPRED +RUN: Stage-2:MAPRED +RUN: Stage-10:CONDITIONAL +RUN: Stage-8:MAPRED +RUN: Stage-4:MAPRED +RUN: Stage-5:MAPRED 128 1 146 1 150 1 @@ -1895,6 +1914,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### +RUN: Stage-2:MAPRED +RUN: Stage-3:MAPRED 128 1 146 1 150 1 @@ -1919,6 +1940,7 @@ CREATE TABLE part_table(key string, value string) PARTITIONED BY (partitionId in POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@part_table +RUN: Stage-0:DDL PREHOOK: query: INSERT OVERWRITE TABLE part_table PARTITION (partitionId=1) SELECT key, value FROM src ORDER BY key, value LIMIT 100 PREHOOK: type: QUERY @@ -1931,6 +1953,9 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@part_table@partitionid=1 POSTHOOK: Lineage: part_table PARTITION(partitionid=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: part_table PARTITION(partitionid=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-1:MAPRED +RUN: Stage-0:MOVE +RUN: Stage-2:STATS PREHOOK: query: INSERT OVERWRITE TABLE part_table PARTITION (partitionId=2) SELECT key, value FROM src1 ORDER BY key, value PREHOOK: type: QUERY @@ -1945,6 +1970,9 @@ POSTHOOK: Lineage: part_table PARTITION(partitionid=1).key SIMPLE [(src)src.Fiel POSTHOOK: Lineage: part_table PARTITION(partitionid=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: part_table PARTITION(partitionid=2).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: part_table PARTITION(partitionid=2).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-1:MAPRED +RUN: Stage-0:MOVE +RUN: Stage-2:STATS PREHOOK: query: EXPLAIN SELECT count(*) FROM part_table x JOIN src1 y ON (x.key = y.key) @@ -2042,6 +2070,7 @@ POSTHOOK: Lineage: part_table PARTITION(partitionid=1).key SIMPLE [(src)src.Fiel POSTHOOK: Lineage: part_table PARTITION(partitionid=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: part_table PARTITION(partitionid=2).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: part_table PARTITION(partitionid=2).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-2:MAPRED 121 PREHOOK: query: -- HIVE-5891 Alias conflict when merging multiple mapjoin tasks into their common -- child mapred task @@ -2310,6 +2339,9 @@ POSTHOOK: Lineage: part_table PARTITION(partitionid=1).key SIMPLE [(src)src.Fiel POSTHOOK: Lineage: part_table PARTITION(partitionid=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: part_table PARTITION(partitionid=2).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: part_table PARTITION(partitionid=2).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-2:MAPRED +RUN: Stage-7:MAPRED +RUN: Stage-4:MAPRED 0 0 0 diff --git ql/src/test/results/clientpositive/subquery_multiinsert.q.out ql/src/test/results/clientpositive/subquery_multiinsert.q.out index cc832b5..8269599 100644 --- ql/src/test/results/clientpositive/subquery_multiinsert.q.out +++ ql/src/test/results/clientpositive/subquery_multiinsert.q.out @@ -1,16 +1,17 @@ -PREHOOK: query: CREATE TABLE src_4( +PREHOOK: query: CREATE TABLE src_4( key STRING, value STRING ) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default -POSTHOOK: query: CREATE TABLE src_4( +POSTHOOK: query: CREATE TABLE src_4( key STRING, value STRING ) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@src_4 +RUN: Stage-0:DDL PREHOOK: query: CREATE TABLE src_5( key STRING, value STRING @@ -24,6 +25,7 @@ POSTHOOK: query: CREATE TABLE src_5( POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@src_5 +RUN: Stage-0:DDL PREHOOK: query: explain from src b INSERT OVERWRITE TABLE src_4 @@ -332,6 +334,15 @@ POSTHOOK: Lineage: src_4.key EXPRESSION [(src)b.FieldSchema(name:key, type:strin POSTHOOK: Lineage: src_4.value EXPRESSION [(src)b.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: src_5.key EXPRESSION [(src)b.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: src_5.value EXPRESSION [(src)b.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-2:MAPRED +RUN: Stage-10:MAPRED +RUN: Stage-0:MOVE +RUN: Stage-4:MAPRED +RUN: Stage-3:STATS +RUN: Stage-5:MAPRED +RUN: Stage-6:MAPRED +RUN: Stage-1:MOVE +RUN: Stage-7:STATS PREHOOK: query: select * from src_4 PREHOOK: type: QUERY PREHOOK: Input: default@src_4 @@ -844,6 +855,17 @@ POSTHOOK: Lineage: src_5.key EXPRESSION [(src)b.FieldSchema(name:key, type:strin POSTHOOK: Lineage: src_5.key EXPRESSION [(src)b.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: src_5.value EXPRESSION [(src)b.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: src_5.value EXPRESSION [(src)b.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-10:MAPRED +RUN: Stage-17:MAPREDLOCAL +RUN: Stage-14:MAPRED +RUN: Stage-0:MOVE +RUN: Stage-13:CONDITIONAL +RUN: Stage-3:STATS +RUN: Stage-12:MAPRED +RUN: Stage-15:MAPREDLOCAL +RUN: Stage-6:MAPRED +RUN: Stage-1:MOVE +RUN: Stage-7:STATS PREHOOK: query: select * from src_4 PREHOOK: type: QUERY PREHOOK: Input: default@src_4