diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java index 0f9e86b..e3b293a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java @@ -775,51 +775,57 @@ public MapJoinOperator convertJoinMapJoin(JoinOperator joinOp, OptimizeTezProcCo return mapJoinOp; } - // Remove any semijoin branch associated with mapjoin's parent's operator - // pipeline which can cause a cycle after mapjoin optimization. + // Remove any semijoin branch associated with hashjoin's parent's operator + // pipeline which can cause a cycle after hashjoin optimization. private void removeCycleCreatingSemiJoinOps(MapJoinOperator mapjoinOp, Operator parentSelectOpOfBigTable, ParseContext parseContext) throws SemanticException { - boolean semiJoinCycle = false; - ReduceSinkOperator rs = null; - TableScanOperator ts = null; + Map semiJoinMap = + new HashMap(); for (Operator op : parentSelectOpOfBigTable.getChildOperators()) { if (!(op instanceof SelectOperator)) { continue; } - while (op.getChildOperators().size() > 0 ) { + while (op.getChildOperators().size() > 0) { op = op.getChildOperators().get(0); - if (!(op instanceof ReduceSinkOperator)) { - continue; - } - rs = (ReduceSinkOperator) op; - ts = parseContext.getRsOpToTsOpMap().get(rs); - if (ts == null) { + } + + // If not ReduceSink Op, skip + if (!(op instanceof ReduceSinkOperator)) { + continue; + } + + ReduceSinkOperator rs = (ReduceSinkOperator) op; + TableScanOperator ts = parseContext.getRsOpToTsOpMap().get(rs); + if (ts == null) { + // skip, no semijoin branch + continue; + } + + // Found a semijoin branch. + for (Operator parent : mapjoinOp.getParentOperators()) { + if (!(parent instanceof ReduceSinkOperator)) { continue; } - for (Operator parent : mapjoinOp.getParentOperators()) { - if (!(parent instanceof ReduceSinkOperator)) { - continue; - } - Set tsOps = OperatorUtils.findOperatorsUpstream(parent, - TableScanOperator.class); - for (TableScanOperator parentTS : tsOps) { - // If the parent is same as the ts, then we have a cycle. - if (ts == parentTS) { - semiJoinCycle = true; - break; - } + Set tsOps = OperatorUtils.findOperatorsUpstream(parent, + TableScanOperator.class); + for (TableScanOperator parentTS : tsOps) { + // If the parent is same as the ts, then we have a cycle. + if (ts == parentTS) { + semiJoinMap.put(rs, ts); + break; } } } } - - // By design there can be atmost 1 such cycle. - if (semiJoinCycle) { - GenTezUtils.removeBranch(rs); - GenTezUtils.removeSemiJoinOperator(parseContext, rs, ts); + if (semiJoinMap.size() > 0) { + for (ReduceSinkOperator rs : semiJoinMap.keySet()) { + GenTezUtils.removeBranch(rs); + GenTezUtils.removeSemiJoinOperator(parseContext, rs, + semiJoinMap.get(rs)); + } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index aee74ad..7f5fdff 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -265,7 +265,6 @@ public static void removeUnionOperators(GenTezProcContext context, BaseWork work for (ReduceSinkOperator rs : rsOpToTsOpMap.keySet()) { if (rsOpToTsOpMap.get(rs) == orig) { rsOpToTsOpMap.put(rs, (TableScanOperator) newRoot); - break; } } } @@ -569,7 +568,8 @@ public static void removeSemiJoinOperator(ParseContext context, TypeInfoFactory.booleanTypeInfo, Boolean.TRUE); DynamicValuePredicateContext filterDynamicValuePredicatesCollection = new DynamicValuePredicateContext(); - collectDynamicValuePredicates(((FilterOperator)(ts.getChildOperators().get(0))).getConf().getPredicate(), + FilterDesc filterDesc = ((FilterOperator)(ts.getChildOperators().get(0))).getConf(); + collectDynamicValuePredicates(filterDesc.getPredicate(), filterDynamicValuePredicatesCollection); for (ExprNodeDesc nodeToRemove : filterDynamicValuePredicatesCollection .childParentMapping.keySet()) { @@ -594,8 +594,8 @@ public static void removeSemiJoinOperator(ParseContext context, ExprNodeDesc nodeParent = filterDynamicValuePredicatesCollection .childParentMapping.get(nodeToRemove); if (nodeParent == null) { - // This was the only predicate, set filter expression to null - ts.getConf().setFilterExpr(null); + // This was the only predicate, set filter expression to const + filterDesc.setPredicate(constNode); } else { int i = nodeParent.getChildren().indexOf(nodeToRemove); nodeParent.getChildren().remove(i);