diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java index b8c01020b7..274df5e50f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java @@ -220,23 +220,34 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Obje } String tableAlias = (op == null ? "" : ((TableScanOperator) op).getConf().getAlias()); - keyBaseAlias = ctx.generator.getOperatorId() + "_" + tableAlias + "_" + column; - - Map hints = parseContext.getSemiJoinHints(); - if (hints != null) { - // If hints map has no entry that would imply that user enforced - // no runtime filtering. - if (hints.size() > 0) { - SemiJoinHint sjHint = hints.get(tableAlias); - semiJoinAttempted = generateSemiJoinOperatorPlan( - ctx, parseContext, ts, keyBaseAlias, sjHint); - if (!semiJoinAttempted && sjHint != null) { - throw new SemanticException("The user hint to enforce semijoin failed required conditions"); + StringBuilder internalColNameBuilder = new StringBuilder(); + StringBuilder colNameBuilder = new StringBuilder(); + if (getColumnName(ctx, internalColNameBuilder, colNameBuilder)) { + String colName = colNameBuilder.length() > 0 ? colNameBuilder.toString() : null; + keyBaseAlias = ctx.generator.getOperatorId() + "_" + tableAlias + "_" + + (colName != null ? colName : column); + Map hints = parseContext.getSemiJoinHints(); + if (hints != null) { + if (hints.size() > 0) { + SemiJoinHint sjHint = hints.get(tableAlias); + if (sjHint != null && sjHint.getColName() != null && + colName != null && !colName.equals(sjHint.getColName())) { + LOG.debug("Deepak : Removed hint due to column mismatch + Col = " + colName + " hint column = " + sjHint.getColName()); + sjHint = null; + } + semiJoinAttempted = generateSemiJoinOperatorPlan( + ctx, parseContext, ts, keyBaseAlias, + internalColNameBuilder.toString(), colName, sjHint); + if (!semiJoinAttempted && sjHint != null) { + throw new SemanticException("The user hint to enforce semijoin failed required conditions"); + } } + } else { + // fallback to regular logic + semiJoinAttempted = generateSemiJoinOperatorPlan( + ctx, parseContext, ts, keyBaseAlias, + internalColNameBuilder.toString(), colName, null); } - } else { - semiJoinAttempted = generateSemiJoinOperatorPlan( - ctx, parseContext, ts, keyBaseAlias, null); } } } @@ -285,6 +296,41 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Obje return false; } + // Given a key, find the corresponding column name. + private boolean getColumnName(DynamicListContext ctx, StringBuilder internalColName, + StringBuilder colName) { + ExprNodeDesc exprNodeDesc = ctx.generator.getConf().getKeyCols().get(ctx.desc.getKeyIndex()); + + while (!(exprNodeDesc instanceof ExprNodeColumnDesc) && + (exprNodeDesc.getChildren() != null)) { + exprNodeDesc = exprNodeDesc.getChildren().get(0); + } + + if (!(exprNodeDesc instanceof ExprNodeColumnDesc)) { + return false; + } + + internalColName.append(((ExprNodeColumnDesc) exprNodeDesc).getColumn()); + Operator parentOfRS = ctx.generator.getParentOperators().get(0); + if (!(parentOfRS instanceof SelectOperator)) { + return true; + } + + ExprNodeDesc expr = parentOfRS.getColumnExprMap().get(internalColName.toString()); + while (!(expr instanceof ExprNodeColumnDesc) && + (expr.getChildren() != null)) { + expr = expr.getChildren().get(0); + } + + if (!(expr instanceof ExprNodeColumnDesc)) { + return false; + } + + ExprNodeColumnDesc colExpr = (ExprNodeColumnDesc) expr; + colName.append(ExprNodeDescUtils.extractColName(colExpr)); + return true; + } + private void replaceExprNode(DynamicListContext ctx, FilterDesc desc, ExprNodeDesc node) { if (ctx.grandParent == null) { desc.setPredicate(node); @@ -400,7 +446,8 @@ private void generateEventOperatorPlan(DynamicListContext ctx, ParseContext pars // Generates plan for min/max when dynamic partition pruning is ruled out. private boolean generateSemiJoinOperatorPlan(DynamicListContext ctx, ParseContext parseContext, - TableScanOperator ts, String keyBaseAlias, SemiJoinHint sjHint) throws SemanticException { + TableScanOperator ts, String keyBaseAlias, String internalColName, + String colName, SemiJoinHint sjHint) throws SemanticException { // If semijoin hint is enforced, make sure hint is provided if (parseContext.getConf().getBoolVar(ConfVars.TEZ_DYNAMIC_SEMIJOIN_REDUCTION_HINT_ONLY) @@ -414,38 +461,7 @@ private boolean generateSemiJoinOperatorPlan(DynamicListContext ctx, ParseContex // we need the expr that generated the key of the reduce sink ExprNodeDesc key = ctx.generator.getConf().getKeyCols().get(ctx.desc.getKeyIndex()); - String internalColName = null; - ExprNodeDesc exprNodeDesc = key; - // Find the ExprNodeColumnDesc - while (!(exprNodeDesc instanceof ExprNodeColumnDesc) && - (exprNodeDesc.getChildren() != null)) { - exprNodeDesc = exprNodeDesc.getChildren().get(0); - } - - if (!(exprNodeDesc instanceof ExprNodeColumnDesc)) { - // No column found! - // Bail out - return false; - } - - internalColName = ((ExprNodeColumnDesc) exprNodeDesc).getColumn(); - if (parentOfRS instanceof SelectOperator) { - // Make sure the semijoin branch is not on partition column. - ExprNodeDesc expr = parentOfRS.getColumnExprMap().get(internalColName); - while (!(expr instanceof ExprNodeColumnDesc) && - (expr.getChildren() != null)) { - expr = expr.getChildren().get(0); - } - - if (!(expr instanceof ExprNodeColumnDesc)) { - // No column found! - // Bail out - return false; - } - - ExprNodeColumnDesc colExpr = (ExprNodeColumnDesc) expr; - String colName = ExprNodeDescUtils.extractColName(colExpr); - + if (colName != null) { // Fetch the TableScan Operator. Operator op = parentOfRS.getParentOperators().get(0); while (op != null && !(op instanceof TableScanOperator)) {