diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java index c0b7a32..68fe178 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hive.ql.exec.FunctionInfo; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.GroupByOperator; +import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.ScriptOperator; import org.apache.hadoop.hive.ql.exec.SelectOperator; @@ -105,6 +106,7 @@ private final boolean doSkipUdfCheck; private final boolean arePermanentFnsAllowed; private final boolean shouldUber; + private List mapJoinOpList; public LlapDecisionDispatcher(PhysicalContext pctx, LlapMode mode) { conf = pctx.getConf(); @@ -112,6 +114,7 @@ public LlapDecisionDispatcher(PhysicalContext pctx, LlapMode mode) { arePermanentFnsAllowed = HiveConf.getBoolVar(conf, ConfVars.LLAP_ALLOW_PERMANENT_FNS); // Don't user uber in "all" mode - everything can go into LLAP, which is better than uber. shouldUber = HiveConf.getBoolVar(conf, ConfVars.LLAP_AUTO_ALLOW_UBER) && (mode != all); + mapJoinOpList = new ArrayList(); } @Override @@ -130,9 +133,18 @@ public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) private void handleWork(TezWork tezWork, BaseWork work) throws SemanticException { - if (evaluateWork(tezWork, work)) { + boolean workCanBeDoneInLlap = evaluateWork(tezWork, work); + LOG.debug( + "Work " + work + " " + (workCanBeDoneInLlap ? "can" : "cannot") + " be done in LLAP"); + if (workCanBeDoneInLlap) { + for (MapJoinOperator graceMapJoinOp : mapJoinOpList) { + LOG.debug( + "Disabling hybrid grace hash join in case of LLAP and non-dynamic partition hash join."); + graceMapJoinOp.getConf().setHybridHashJoin(false); + } convertWork(tezWork, work); } + mapJoinOpList.clear(); } private void convertWork(TezWork tezWork, BaseWork work) @@ -177,12 +189,13 @@ private boolean evaluateWork(TezWork tezWork, BaseWork work) // if mode is all just run it if (mode == all) { + LOG.info("LLAP mode set to 'all' so can convert any work."); return true; } // if map mode run iff work is map work if (mode == map) { - return work instanceof MapWork; + return (work instanceof MapWork); } // --- From here we evaluate the auto mode @@ -337,6 +350,20 @@ public Object process(Node n, Stack s, NodeProcessorCtx c, } }); + opRules.put( + new RuleRegExp("Disable grace hash join if LLAP mode and not dynamic partition hash join", + MapJoinOperator.getOperatorName() + "%"), new NodeProcessor() { + @Override + public Object process(Node n, Stack s, NodeProcessorCtx c, Object... os) { + MapJoinOperator mapJoinOp = (MapJoinOperator) n; + if (mapJoinOp.getConf().isHybridHashJoin() + && !(mapJoinOp.getConf().isDynamicPartitionHashJoin())) { + mapJoinOpList.add((MapJoinOperator) n); + } + return new Boolean(true); + } + }); + return opRules; } diff --git ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out index f281d44..be531a1 100644 --- ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out @@ -196,7 +196,6 @@ STAGE PLANS: input vertices: 1 Map 3 Statistics: Num rows: 4505 Data size: 968719 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -439,7 +438,6 @@ STAGE PLANS: input vertices: 1 Map 3 Statistics: Num rows: 13516 Data size: 2906160 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -669,7 +667,6 @@ STAGE PLANS: input vertices: 1 Map 3 Statistics: Num rows: 13516 Data size: 2906160 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -943,7 +940,6 @@ STAGE PLANS: input vertices: 1 Map 3 Statistics: Num rows: 577 Data size: 13721 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1190,7 +1186,6 @@ STAGE PLANS: input vertices: 1 Map 3 Statistics: Num rows: 577 Data size: 13721 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash diff --git ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out index 95da4c5..b8f24c4 100644 --- ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out +++ ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out @@ -211,7 +211,6 @@ STAGE PLANS: 0 Map 1 2 Map 4 Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -512,7 +511,6 @@ STAGE PLANS: 2 Map 4 3 Map 5 Statistics: Num rows: 6600 Data size: 70118 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -948,7 +946,6 @@ STAGE PLANS: 0 Map 1 2 Map 6 Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1010,7 +1007,6 @@ STAGE PLANS: 0 Map 7 2 Map 10 Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1392,7 +1388,6 @@ STAGE PLANS: 0 Map 1 2 Map 4 Statistics: Num rows: 1465 Data size: 15565 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Map Join Operator condition map: Inner Join 0 to 1 @@ -1405,7 +1400,6 @@ STAGE PLANS: 1 Map 5 2 Map 6 Statistics: Num rows: 3223 Data size: 34243 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash