diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java index 737d9c3..0915ff0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hive.ql.exec.FunctionInfo; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.GroupByOperator; +import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.ScriptOperator; import org.apache.hadoop.hive.ql.exec.SelectOperator; @@ -103,6 +104,7 @@ private final boolean doSkipUdfCheck; private final boolean arePermanentFnsAllowed; private final boolean shouldUber; + private final List mapJoinOpList = new ArrayList(); public LlapDecisionDispatcher(PhysicalContext pctx, LlapMode mode) { conf = pctx.getConf(); @@ -128,7 +130,16 @@ public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) private void handleWork(TezWork tezWork, BaseWork work) throws SemanticException { - if (evaluateWork(tezWork, work)) { + boolean workCanBeDoneInLlap = evaluateWork(tezWork, work); + LOG.info( + "Work " + work + " " + (workCanBeDoneInLlap ? "can" : "cannot") + " be done in LLAP"); + if (workCanBeDoneInLlap) { + for (MapJoinOperator graceMapJoinOp : mapJoinOpList) { + LOG.info( + "Disabling hybrid grace hash join in case of LLAP and non-dynamic partition hash join."); + graceMapJoinOp.getConf().setHybridHashJoin(false); + } + mapJoinOpList.clear(); convertWork(tezWork, work); } } @@ -175,12 +186,16 @@ private boolean evaluateWork(TezWork tezWork, BaseWork work) // if mode is all just run it if (mode == all) { + LOG.info("LLAP mode set to 'all' so can convert any work."); return true; } // if map mode run iff work is map work if (mode == map) { - return work instanceof MapWork; + boolean retval = (work instanceof MapWork); + LOG.info("Work is of type " + work.getClass().getCanonicalName() + ". " + + (retval ? "NOT allowed" : "Allowed") + " in llap map-only mode."); + return retval; } // --- From here we evaluate the auto mode @@ -333,6 +348,20 @@ public Object process(Node n, Stack s, NodeProcessorCtx c, } }); + opRules.put( + new RuleRegExp("Disable grace hash join if LLAP mode and not dynamic partition hash join", + MapJoinOperator.getOperatorName() + "%"), new NodeProcessor() { + @Override + public Object process(Node n, Stack s, NodeProcessorCtx c, Object... os) { + MapJoinOperator mapJoinOp = (MapJoinOperator) n; + if (mapJoinOp.getConf().isHybridHashJoin() + && !(mapJoinOp.getConf().isDynamicPartitionHashJoin())) { + mapJoinOpList.add((MapJoinOperator) n); + } + return new Boolean(true); + } + }); + return opRules; } diff --git ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out index aaa8425..4f9d56f 100644 --- ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out @@ -194,7 +194,6 @@ STAGE PLANS: input vertices: 1 Map 3 Statistics: Num rows: 4505 Data size: 968719 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -433,7 +432,6 @@ STAGE PLANS: input vertices: 1 Map 3 Statistics: Num rows: 13516 Data size: 2906160 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -659,7 +657,6 @@ STAGE PLANS: input vertices: 1 Map 3 Statistics: Num rows: 13516 Data size: 2906160 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -929,7 +926,6 @@ STAGE PLANS: input vertices: 1 Map 3 Statistics: Num rows: 577 Data size: 13721 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1172,7 +1168,6 @@ STAGE PLANS: input vertices: 1 Map 3 Statistics: Num rows: 577 Data size: 13721 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash diff --git ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out index 7a00162..c3cf3a8 100644 --- ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out +++ ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out @@ -207,7 +207,6 @@ STAGE PLANS: 0 Map 1 2 Map 4 Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -501,7 +500,6 @@ STAGE PLANS: 2 Map 4 3 Map 5 Statistics: Num rows: 6600 Data size: 70118 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -926,7 +924,6 @@ STAGE PLANS: 0 Map 1 2 Map 6 Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -985,7 +982,6 @@ STAGE PLANS: 0 Map 7 2 Map 10 Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -1360,7 +1356,6 @@ STAGE PLANS: 0 Map 1 2 Map 4 Statistics: Num rows: 1465 Data size: 15565 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Map Join Operator condition map: Inner Join 0 to 1 @@ -1373,7 +1368,6 @@ STAGE PLANS: 1 Map 5 2 Map 6 Statistics: Num rows: 3223 Data size: 34243 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash