diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index c5f39d3..fc2063d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -301,7 +301,17 @@ protected AnalyzeRewriteContext analyzeRewrite; private CreateTableDesc tableDesc; - + /* + * Used to fix filter bug of UNION ALL when + * hive.ppd.remove.duplicatefilters=true and filter condition is type + * incompatible column in HIVE-11880 + */ + private static Boolean isUnion = false; + private static Boolean existIncompatibleTypeOfUnionOperator = false; + private static Boolean isIncompatibleUnionColConstant = false; + private static Boolean isFilterColSameWithIncompatibleColOfUnionOperator = false; + private static List incompatibleColOfUnionOperator = new ArrayList(); + public static Boolean needRemoveDuplicateOperatorOfUnion = true; static class Phase1Ctx { String dest; int nextNum; @@ -2806,6 +2816,21 @@ private Operator genFilterPlan(QB qb, ASTNode condn, Operator input, boolean use OpParseContext inputCtx = opParseCtx.get(input); RowResolver inputRR = inputCtx.getRowResolver(); + ExprNodeDesc exprNodeDesc = genExprNodeDesc(condn, inputRR); + // check if exists filter column which is same with type incompatible column + // of Union Operator + for (String filterCol : exprNodeDesc.getCols()) { + for (int i = 0; i < incompatibleColOfUnionOperator.size(); i++) { + if (incompatibleColOfUnionOperator.get(i).equals(filterCol)) { + isFilterColSameWithIncompatibleColOfUnionOperator = true; + } + } + } + if (isUnion && existIncompatibleTypeOfUnionOperator + && isIncompatibleUnionColConstant + && isFilterColSameWithIncompatibleColOfUnionOperator) { + needRemoveDuplicateOperatorOfUnion = false; + } Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( new FilterDesc(genExprNodeDesc(condn, inputRR, useCaching), false), new RowSchema( inputRR.getColumnInfos()), input), inputRR); @@ -9012,7 +9037,7 @@ private Operator genPostGroupByBodyPlan(Operator curr, String dest, QB qb, private Operator genUnionPlan(String unionalias, String leftalias, Operator leftOp, String rightalias, Operator rightOp) throws SemanticException { - + isUnion = true; // Currently, the unions are not merged - each union has only 2 parents. So, // a n-way union will lead to (n-1) union operators. // This can be easily merged into 1 union @@ -9038,6 +9063,18 @@ private Operator genUnionPlan(String unionalias, String leftalias, ColumnInfo rInfo = rEntry.getValue(); String field = lEntry.getKey(); // use left alias (~mysql, postgresql) + // check if exists type incompatible column and if it is Constant + if (lInfo.getType() != rInfo.getType()) { + existIncompatibleTypeOfUnionOperator = true; + if (lInfo.getObjectInspector() instanceof ConstantObjectInspector) { + isIncompatibleUnionColConstant = true; + incompatibleColOfUnionOperator.add(lInfo.getInternalName()); + } + if (rInfo.getObjectInspector() instanceof ConstantObjectInspector) { + isIncompatibleUnionColConstant = true; + incompatibleColOfUnionOperator.add(rInfo.getInternalName()); + } + } // try widening conversion, otherwise fail union TypeInfo commonTypeInfo = FunctionRegistry.getCommonClassForUnionAll(lInfo.getType(), rInfo.getType()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java index dbd021b..b767220 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler; import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.WindowingSpec.Direction; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -426,9 +427,12 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } if (HiveConf.getBoolVar(owi.getParseContext().getConf(), HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) { - // add this filter for deletion, if it does not have non-final candidates - if (ewi.getNonFinalCandidates().values().isEmpty()) { - owi.addCandidateFilterOp((FilterOperator)op); + // check if need to remove duplicate filters + if (SemanticAnalyzer.needRemoveDuplicateOperatorOfUnion) { + // add this filter for deletion, if it does not have non-final candidates + if (ewi.getNonFinalCandidates().values().isEmpty()) { + owi.addCandidateFilterOp((FilterOperator) op); + } } } logExpr(nd, ewi);