diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java index 8f751260a0..a9d6a5c942 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.optimizer.metainfo.annotation; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; import java.util.*; import java.util.Map.Entry; @@ -124,18 +126,25 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, continue; } // Fetch the column expression. There should be atleast one. - Map colMap = new HashMap<>(); + Multimap colMap = ArrayListMultimap.create(); boolean found = false; ExprNodeDescUtils.getExprNodeColumnDesc(entry.getValue(), colMap); for (Integer hashCode : colMap.keySet()) { - ExprNodeColumnDesc expr = (ExprNodeColumnDesc) colMap.get(hashCode); - if (expr.getColumn().equals(col)) { - bucketCols.add(entry.getKey()); - found = true; + Collection exprs = colMap.get(hashCode); + for (ExprNodeColumnDesc expr : exprs) { + if (expr.getColumn().equals(col)) { + bucketCols.add(entry.getKey()); + found = true; + break; + } + } + if (found) { break; } } - if (found) break; + if (found) { + break; + } } // column exprmap. } // cols } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 8d9718f2c8..5ed0bd500e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -5174,12 +5174,12 @@ public Object post(Object t) { List col_list, RowResolver inputRR) { // Build a map of Hive column Names (ExprNodeColumnDesc Name) // to the positions of those projections in the input - Map hashCodeTocolumnDescMap = new HashMap(); + Multimap hashCodeTocolumnDescMap = ArrayListMultimap.create(); ExprNodeDescUtils.getExprNodeColumnDesc(col_list, hashCodeTocolumnDescMap); ImmutableMap.Builder hiveColNameToInputPosMapBuilder = new ImmutableMap.Builder(); String exprNodecolName; - for (ExprNodeDesc exprDesc : hashCodeTocolumnDescMap.values()) { - exprNodecolName = ((ExprNodeColumnDesc) exprDesc).getColumn(); + for (ExprNodeColumnDesc exprDesc : hashCodeTocolumnDescMap.values()) { + exprNodecolName = exprDesc.getColumn(); hiveColNameToInputPosMapBuilder.put(exprNodecolName, inputRR.getPosition(exprNodecolName)); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 45482764fd..df3c6b9db1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -20,6 +20,8 @@ import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVESTATSDBCLASS; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; import java.io.FileNotFoundException; import java.io.IOException; import java.security.AccessControlException; @@ -3514,7 +3516,7 @@ private Operator genNotNullFilterForJoinSourcePlan(QB qb, Operator input, if (joinKeys == null || joinKeys.length == 0) { return input; } - Map hashes = new HashMap(); + Multimap hashes = ArrayListMultimap.create(); if (input instanceof FilterOperator) { ExprNodeDescUtils.getExprNodeColumnDesc(Arrays.asList(((FilterDesc)input.getConf()).getPredicate()), hashes); } @@ -3527,7 +3529,14 @@ private Operator genNotNullFilterForJoinSourcePlan(QB qb, Operator input, // virtual column, since those columns can never be null. continue; } - if(null != hashes.get(joinKeys[i].hashCode())) { + boolean skip = false; + for (ExprNodeColumnDesc node : hashes.get(joinKeys[i].hashCode())) { + if (node.isSame(joinKeys[i])) { + skip = true; + break; + } + } + if (skip) { // there is already a predicate on this src. continue; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java index 29ceb7f981..d887ca8015 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.plan; +import com.google.common.collect.Multimap; +import java.util.Collection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.exec.ColumnInfo; @@ -577,44 +579,51 @@ private static ExprNodeConstantDesc foldConstant(ExprNodeGenericFuncDesc func) { } catch (Exception e) { return null; } - } - - public static void getExprNodeColumnDesc(List exprDescList, - Map hashCodeTocolumnDescMap) { - for (ExprNodeDesc exprNodeDesc : exprDescList) { - getExprNodeColumnDesc(exprNodeDesc, hashCodeTocolumnDescMap); - } - } - - /** - * Get Map of ExprNodeColumnDesc HashCode to ExprNodeColumnDesc. - * - * @param exprDesc - * @param hashCodeToColumnDescMap - * Assumption: If two ExprNodeColumnDesc have same hash code then - * they are logically referring to same projection - */ - public static void getExprNodeColumnDesc(ExprNodeDesc exprDesc, - Map hashCodeToColumnDescMap) { - if (exprDesc instanceof ExprNodeColumnDesc) { - hashCodeToColumnDescMap.put(exprDesc.hashCode(), exprDesc); - } else if (exprDesc instanceof ExprNodeColumnListDesc) { - for (ExprNodeDesc child : exprDesc.getChildren()) { - getExprNodeColumnDesc(child, hashCodeToColumnDescMap); - } - } else if (exprDesc instanceof ExprNodeGenericFuncDesc) { - for (ExprNodeDesc child : exprDesc.getChildren()) { - getExprNodeColumnDesc(child, hashCodeToColumnDescMap); - } - } else if (exprDesc instanceof ExprNodeFieldDesc) { - getExprNodeColumnDesc(((ExprNodeFieldDesc) exprDesc).getDesc(), - hashCodeToColumnDescMap); - } else if( exprDesc instanceof ExprNodeSubQueryDesc) { - getExprNodeColumnDesc(((ExprNodeSubQueryDesc) exprDesc).getSubQueryLhs(), - hashCodeToColumnDescMap); - } + } + + public static void getExprNodeColumnDesc(List exprDescList, + Multimap hashCodeTocolumnDescMap) { + for (ExprNodeDesc exprNodeDesc : exprDescList) { + getExprNodeColumnDesc(exprNodeDesc, hashCodeTocolumnDescMap); + } + } - } + /** + * Get Map of ExprNodeColumnDesc HashCode to ExprNodeColumnDesc. + * + * @param exprDesc + * @param hashCodeToColumnDescMap + */ + public static void getExprNodeColumnDesc(ExprNodeDesc exprDesc, + Multimap hashCodeToColumnDescMap) { + if (exprDesc instanceof ExprNodeColumnDesc) { + Collection nodes = hashCodeToColumnDescMap.get(exprDesc.hashCode()); + boolean insert = true; + for (ExprNodeColumnDesc node : nodes) { + if (node.isSame(exprDesc)) { + insert = false; + break; + } + } + if (insert) { + nodes.add((ExprNodeColumnDesc) exprDesc); + } + } else if (exprDesc instanceof ExprNodeColumnListDesc) { + for (ExprNodeDesc child : exprDesc.getChildren()) { + getExprNodeColumnDesc(child, hashCodeToColumnDescMap); + } + } else if (exprDesc instanceof ExprNodeGenericFuncDesc) { + for (ExprNodeDesc child : exprDesc.getChildren()) { + getExprNodeColumnDesc(child, hashCodeToColumnDescMap); + } + } else if (exprDesc instanceof ExprNodeFieldDesc) { + getExprNodeColumnDesc(((ExprNodeFieldDesc) exprDesc).getDesc(), + hashCodeToColumnDescMap); + } else if(exprDesc instanceof ExprNodeSubQueryDesc) { + getExprNodeColumnDesc(((ExprNodeSubQueryDesc) exprDesc).getSubQueryLhs(), + hashCodeToColumnDescMap); + } + } public static boolean isConstant(ExprNodeDesc value) { if (value instanceof ExprNodeConstantDesc) {