diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 1076dfd..b5aa75a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -301,7 +301,15 @@ protected AnalyzeRewriteContext analyzeRewrite; private CreateTableDesc tableDesc; - + /* + * Used to fix UNION Bug in HIVE-11880 + */ + public static Boolean isUnion = false; + public static Boolean existIncompatibleTypeOfUnionOperator = false; + public static Boolean isIncompatibleUnionColConstant = false; + public static Boolean isFilterColSameWithIncompatibleColOfUnionOperator = false; + public static ArrayList IncompatibleColOfUnionOperator = new ArrayList(); + static class Phase1Ctx { String dest; int nextNum; @@ -2806,6 +2814,15 @@ private Operator genFilterPlan(QB qb, ASTNode condn, Operator input, boolean use OpParseContext inputCtx = opParseCtx.get(input); RowResolver inputRR = inputCtx.getRowResolver(); + ExprNodeDesc exprNodeDesc = genExprNodeDesc(condn, inputRR); + //check isFilterColSameWithIncompatibleColOfUnionOperator for HIVE-11880 + for (String filterCol : exprNodeDesc.getCols()) { + for (int i = 0; i < IncompatibleColOfUnionOperator.size(); i++) { + if (IncompatibleColOfUnionOperator.get(i).equals(filterCol)) { + isFilterColSameWithIncompatibleColOfUnionOperator= true; + } + } + } Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( new FilterDesc(genExprNodeDesc(condn, inputRR, useCaching), false), new RowSchema( inputRR.getColumnInfos()), input), inputRR); @@ -9006,7 +9023,8 @@ private Operator genPostGroupByBodyPlan(Operator curr, String dest, QB qb, private Operator genUnionPlan(String unionalias, String leftalias, Operator leftOp, String rightalias, Operator rightOp) throws SemanticException { - + // whether UNION or not for HIVE-11880 + isUnion = true; // Currently, the unions are not merged - each union has only 2 parents. So, // a n-way union will lead to (n-1) union operators. // This can be easily merged into 1 union @@ -9111,7 +9129,18 @@ private Operator genUnionPlan(String unionalias, String leftalias, rightOp = (Operator) rightOp.getParentOperators().get(0); rightOp.removeChildAndAdoptItsChildren(oldChild); } - + // check if exists IncompatibleTypeOfUnionOperator and if is Constant for HIVE-11880 + if (lInfo.getType() != rInfo.getType()) { + existIncompatibleTypeOfUnionOperator = true; + if (lInfo.getObjectInspector() instanceof ConstantObjectInspector) { + isIncompatibleUnionColConstant = true; + IncompatibleColOfUnionOperator.add(lInfo.getInternalName()); + } + if (rInfo.getObjectInspector() instanceof ConstantObjectInspector) { + isIncompatibleUnionColConstant = true; + IncompatibleColOfUnionOperator.add(rInfo.getInternalName()); + } + } // make right a child of left List> child = new ArrayList>(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java index dbd021b..1f03924 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java @@ -426,9 +426,15 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } if (HiveConf.getBoolVar(owi.getParseContext().getConf(), HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) { - // add this filter for deletion, if it does not have non-final candidates - if (ewi.getNonFinalCandidates().values().isEmpty()) { - owi.addCandidateFilterOp((FilterOperator)op); + //Check if need to remove duplicate filters Of UNION for HIVE-11880 + if (!SemanticAnalyzer.isUnion + || !SemanticAnalyzer.existIncompatibleTypeOfUnionOperator + || !SemanticAnalyzer.isIncompatibleUnionColConstant + || !SemanticAnalyzer.isFilterColSameWithIncompatibleColOfUnionOperator) { + // add this filter for deletion, if it does not have non-final candidates + if (ewi.getNonFinalCandidates().values().isEmpty()) { + owi.addCandidateFilterOp((FilterOperator) op); + } } } logExpr(nd, ewi);