diff --git ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java index 33ad3e8..c7ee0e9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java +++ ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java @@ -37,29 +37,21 @@ public class ExprWalkerInfo implements NodeProcessorCtx { /** Information maintained for an expr while walking an expr tree. */ - private static class ExprInfo { + protected class ExprInfo { /** * true if expr rooted at this node doesn't contain more than one table. * alias */ - public boolean isCandidate = false; + protected boolean isCandidate = false; /** alias that this expression refers to. */ - public String alias = null; + protected String alias = null; /** new expr for this expression. */ - public ExprNodeDesc convertedExpr = null; + protected ExprNodeDesc convertedExpr = null; - public ExprInfo() { - } - public ExprInfo(boolean isCandidate, String alias, ExprNodeDesc replacedNode) { - this.isCandidate = isCandidate; - this.alias = alias; - convertedExpr = replacedNode; - } } - protected static final Log LOG = LogFactory.getLog(OpProcFactory.class - .getName());; + protected static final Log LOG = LogFactory.getLog(OpProcFactory.class.getName()); private Operator op = null; /** @@ -126,105 +118,23 @@ public ExprWalkerInfo(Operator op) { } /** - * @return converted expression for give node. If there is none then returns - * null. - */ - public ExprNodeDesc getConvertedNode(ExprNodeDesc nd) { - ExprInfo ei = exprInfoMap.get(nd); - if (ei == null) { - return null; - } - return ei.convertedExpr; - } - - /** - * adds a replacement node for this expression. - * - * @param oldNode - * original node - * @param newNode - * new node - */ - public void addConvertedNode(ExprNodeDesc oldNode, ExprNodeDesc newNode) { - ExprInfo ei = exprInfoMap.get(oldNode); - if (ei == null) { - ei = new ExprInfo(); - exprInfoMap.put(oldNode, ei); - } - ei.convertedExpr = newNode; - exprInfoMap.put(newNode, new ExprInfo(ei.isCandidate, ei.alias, null)); - } - - /** - * Returns true if the specified expression is pushdown candidate else false. - * - * @param expr - * @return true or false + * Get additional info for a given expression node */ - public boolean isCandidate(ExprNodeDesc expr) { - ExprInfo ei = exprInfoMap.get(expr); - if (ei == null) { - return false; - } - return ei.isCandidate; + public ExprInfo getExprInfo(ExprNodeDesc expr) { + return exprInfoMap.get(expr); } /** - * Marks the specified expr to the specified value. - * - * @param expr - * @param b - * can + * Get additional info for a given expression node if it + * exists, or create a new one and store it if it does not */ - public void setIsCandidate(ExprNodeDesc expr, boolean b) { - ExprInfo ei = exprInfoMap.get(expr); - if (ei == null) { - ei = new ExprInfo(); - exprInfoMap.put(expr, ei); + public ExprInfo addOrGetExprInfo(ExprNodeDesc expr) { + ExprInfo exprInfo = exprInfoMap.get(expr); + if (exprInfo == null) { + exprInfo = new ExprInfo(); + exprInfoMap.put(expr, exprInfo); } - ei.isCandidate = b; - } - - /** - * Returns the alias of the specified expr. - * - * @param expr - * @return The alias of the expression - */ - public String getAlias(ExprNodeDesc expr) { - ExprInfo ei = exprInfoMap.get(expr); - if (ei == null) { - return null; - } - return ei.alias; - } - - /** - * Adds the specified alias to the specified expr. - * - * @param expr - * @param alias - */ - public void addAlias(ExprNodeDesc expr, String alias) { - if (alias == null) { - return; - } - ExprInfo ei = exprInfoMap.get(expr); - if (ei == null) { - ei = new ExprInfo(); - exprInfoMap.put(expr, ei); - } - ei.alias = alias; - } - - /** - * Adds the specified expr as the top-most pushdown expr (ie all its children - * can be pushed). - * - * @param expr - */ - public void addFinalCandidate(ExprNodeDesc expr) { - addFinalCandidate(getAlias(expr), expr); + return exprInfo; } public void addFinalCandidate(String alias, ExprNodeDesc expr) { @@ -277,8 +187,7 @@ public boolean hasAnyCandidates() { * * @param expr */ - public void addNonFinalCandidate(ExprNodeDesc expr) { - String alias = getAlias(expr); + public void addNonFinalCandidate(String alias, ExprNodeDesc expr) { if (nonFinalPreds.get(alias) == null) { nonFinalPreds.put(alias, new ArrayList()); } diff --git ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java index 6a1bef9..869ead9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java @@ -38,8 +38,6 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleExactMatch; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.lib.TypeRule; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -47,6 +45,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.ql.ppd.ExprWalkerInfo.ExprInfo; /** * Expression factory for predicate pushdown processing. Each processor @@ -55,8 +54,7 @@ */ public final class ExprWalkerProcFactory { - private static final Log LOG = LogFactory - .getLog(ExprWalkerProcFactory.class.getName()); + private static final Log LOG = LogFactory.getLog(ExprWalkerProcFactory.class.getName()); /** * ColumnExprProcessor. @@ -80,6 +78,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, tabAlias = ci.getTabAlias(); } + ExprInfo colExprInfo = null; boolean isCandidate = true; if (op.getColumnExprMap() != null) { // replace the output expression with the input expression so that @@ -88,7 +87,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, if (exp == null) { // means that expression can't be pushed either because it is value in // group by - ctx.setIsCandidate(colref, false); + colExprInfo = ctx.addOrGetExprInfo(colref); + colExprInfo.isCandidate = false; return false; } else { if (exp instanceof ExprNodeGenericFuncDesc) { @@ -99,16 +99,25 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, tabAlias = column.getTabAlias(); } } - ctx.addConvertedNode(colref, exp); - ctx.setIsCandidate(exp, isCandidate); - ctx.addAlias(exp, tabAlias); + colExprInfo = ctx.addOrGetExprInfo(colref); + colExprInfo.convertedExpr = exp; + ExprInfo expInfo = ctx.addOrGetExprInfo(exp); + expInfo.isCandidate = isCandidate; + if (tabAlias != null) { + expInfo.alias = tabAlias; + } else { + expInfo.alias = colExprInfo.alias; + } } else { if (ci == null) { return false; } - ctx.addAlias(colref, tabAlias); + if (tabAlias != null) { + colExprInfo = ctx.addOrGetExprInfo(colref); + colExprInfo.alias = tabAlias; + } } - ctx.setIsCandidate(colref, isCandidate); + colExprInfo.isCandidate = isCandidate; return isCandidate; } @@ -130,27 +139,36 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, boolean isCandidate = true; assert (nd.getChildren().size() == 1); ExprNodeDesc ch = (ExprNodeDesc) nd.getChildren().get(0); - ExprNodeDesc newCh = ctx.getConvertedNode(ch); + ExprInfo chExprInfo = ctx.getExprInfo(ch); + ExprNodeDesc newCh = chExprInfo != null ? chExprInfo.convertedExpr : null; if (newCh != null) { expr.setDesc(newCh); ch = newCh; + chExprInfo = ctx.getExprInfo(ch); } - String chAlias = ctx.getAlias(ch); - isCandidate = isCandidate && ctx.isCandidate(ch); + String chAlias; + if (chExprInfo != null) { + chAlias = chExprInfo.alias; + isCandidate = isCandidate && chExprInfo.isCandidate; + } else { + chAlias = null; + isCandidate = false; + } // need to iterate through all children even if one is found to be not a // candidate // in case if the other children could be individually pushed up if (isCandidate && chAlias != null) { if (alias == null) { alias = chAlias; - } else if (!chAlias.equalsIgnoreCase(alias)) { - isCandidate = false; } } - ctx.addAlias(expr, alias); - ctx.setIsCandidate(expr, isCandidate); + ExprInfo exprInfo = ctx.addOrGetExprInfo(expr); + if (alias != null) { + exprInfo.alias = alias; + } + exprInfo.isCandidate = isCandidate; return isCandidate; } @@ -172,7 +190,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, if (!FunctionRegistry.isDeterministic(expr.getGenericUDF())) { // this GenericUDF can't be pushed down - ctx.setIsCandidate(expr, false); + ExprInfo exprInfo = ctx.addOrGetExprInfo(expr); + exprInfo.isCandidate = false; ctx.setDeterministic(false); return false; } @@ -180,14 +199,22 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, boolean isCandidate = true; for (int i = 0; i < nd.getChildren().size(); i++) { ExprNodeDesc ch = (ExprNodeDesc) nd.getChildren().get(i); - ExprNodeDesc newCh = ctx.getConvertedNode(ch); + ExprInfo chExprInfo = ctx.getExprInfo(ch); + ExprNodeDesc newCh = chExprInfo != null ? chExprInfo.convertedExpr : null; if (newCh != null) { expr.getChildren().set(i, newCh); ch = newCh; + chExprInfo = ctx.getExprInfo(ch); } - String chAlias = ctx.getAlias(ch); - isCandidate = isCandidate && ctx.isCandidate(ch); + String chAlias; + if (chExprInfo != null) { + chAlias = chExprInfo.alias; + isCandidate = isCandidate && chExprInfo.isCandidate; + } else { + chAlias = null; + isCandidate = false; + } // need to iterate through all children even if one is found to be not a // candidate // in case if the other children could be individually pushed up @@ -203,8 +230,11 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, break; } } - ctx.addAlias(expr, alias); - ctx.setIsCandidate(expr, isCandidate); + ExprInfo exprInfo = ctx.addOrGetExprInfo(expr); + if (alias != null) { + exprInfo.alias = alias; + } + exprInfo.isCandidate = isCandidate; return isCandidate; } @@ -219,7 +249,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { ExprWalkerInfo ctx = (ExprWalkerInfo) procCtx; - ctx.setIsCandidate((ExprNodeDesc) nd, true); + ExprInfo exprInfo = ctx.addOrGetExprInfo((ExprNodeDesc) nd); + exprInfo.isCandidate = true; return true; } } @@ -324,12 +355,13 @@ private static void extractFinalCandidates(ExprNodeDesc expr, return; } - if (ctx.isCandidate(expr)) { - ctx.addFinalCandidate(expr); + ExprInfo exprInfo = ctx.getExprInfo(expr); + if (exprInfo != null && exprInfo.isCandidate) { + ctx.addFinalCandidate(exprInfo.alias, expr); return; } else if (!FunctionRegistry.isOpAnd(expr) && HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) { - ctx.addNonFinalCandidate(expr); + ctx.addNonFinalCandidate(exprInfo != null ? exprInfo.alias : null, expr); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java index 6f9df53..ef1fbe8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java @@ -66,6 +66,7 @@ import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef; import org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef; import org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef; +import org.apache.hadoop.hive.ql.ppd.ExprWalkerInfo.ExprInfo; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFDenseRank.GenericUDAFDenseRankEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFLead.GenericUDAFLeadEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFRank.GenericUDAFRankEvaluator; @@ -483,8 +484,11 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, prunePreds.getFinalCandidates().get(alias)) { // add expr to the list of predicates rejected from further pushing // so that we know to add it in createFilter() - prunePreds.addAlias(expr, alias); - prunePreds.addNonFinalCandidate(expr); + ExprInfo exprInfo = prunePreds.addOrGetExprInfo(expr); + if (alias != null) { + exprInfo.alias = alias; + } + prunePreds.addNonFinalCandidate(exprInfo.alias, expr); } prunePreds.getFinalCandidates().remove(alias); }