diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 1076dfd..b5aa75a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -301,7 +301,15 @@ protected AnalyzeRewriteContext analyzeRewrite; private CreateTableDesc tableDesc; + /* + * Used to fix filter bug of UNION ALL when hive.ppd.remove.duplicatefilters=true and filter condition is type incompatible column in HIVE-11880 + */ + public static Boolean isUnion = false; + public static Boolean existIncompatibleTypeOfUnionOperator = false; + public static Boolean isIncompatibleUnionColConstant = false; + public static Boolean isFilterColSameWithIncompatibleColOfUnionOperator = false; + public static List incompatibleColOfUnionOperator = new ArrayList(); static class Phase1Ctx { String dest; int nextNum; @@ -2806,6 +2814,15 @@ private Operator genFilterPlan(QB qb, ASTNode condn, Operator input, boolean use OpParseContext inputCtx = opParseCtx.get(input); RowResolver inputRR = inputCtx.getRowResolver(); + ExprNodeDesc exprNodeDesc = genExprNodeDesc(condn, inputRR); + //check if exists filter column which is same with type incompatible column of Union Operator + for (String filterCol : exprNodeDesc.getCols()) { + for (int i = 0; i < incompatibleColOfUnionOperator.size(); i++) { + if (incompatibleColOfUnionOperator.get(i).equals(filterCol)) { + isFilterColSameWithIncompatibleColOfUnionOperator= true; + } + } + } Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( new FilterDesc(genExprNodeDesc(condn, inputRR, useCaching), false), new RowSchema( inputRR.getColumnInfos()), input), inputRR); @@ -9006,7 +9023,8 @@ private Operator genPostGroupByBodyPlan(Operator curr, String dest, QB qb, private Operator genUnionPlan(String unionalias, String leftalias, Operator leftOp, String rightalias, Operator rightOp) throws SemanticException { + isUnion = true; // Currently, the unions are not merged - each union has only 2 parents. So, // a n-way union will lead to (n-1) union operators. // This can be easily merged into 1 union @@ -9111,7 +9129,18 @@ private Operator genUnionPlan(String unionalias, String leftalias, rightOp = (Operator) rightOp.getParentOperators().get(0); rightOp.removeChildAndAdoptItsChildren(oldChild); } + // check if exists type incompatible column and if it is Constant + if (lInfo.getType() != rInfo.getType()) { + existIncompatibleTypeOfUnionOperator = true; + if (lInfo.getObjectInspector() instanceof ConstantObjectInspector) { + isIncompatibleUnionColConstant = true; + incompatibleColOfUnionOperator.add(lInfo.getInternalName()); + } + if (rInfo.getObjectInspector() instanceof ConstantObjectInspector) { + isIncompatibleUnionColConstant = true; + incompatibleColOfUnionOperator.add(rInfo.getInternalName()); + } + } // make right a child of left List> child = new ArrayList>(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java index dbd021b..1f03924 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java @@ -426,9 +426,15 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } if (HiveConf.getBoolVar(owi.getParseContext().getConf(), HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) { - // add this filter for deletion, if it does not have non-final candidates - if (ewi.getNonFinalCandidates().values().isEmpty()) { - owi.addCandidateFilterOp((FilterOperator)op); + //check if need to remove duplicate filters + if (!SemanticAnalyzer.isUnion + || !SemanticAnalyzer.existIncompatibleTypeOfUnionOperator + || !SemanticAnalyzer.isIncompatibleUnionColConstant + || !SemanticAnalyzer.isFilterColSameWithIncompatibleColOfUnionOperator) { + // add this filter for deletion, if it does not have non-final candidates + if (ewi.getNonFinalCandidates().values().isEmpty()) { + owi.addCandidateFilterOp((FilterOperator) op); + } } } logExpr(nd, ewi);