diff --git common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java index 29dc06dca1a..f0b28c720df 100644 --- common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java +++ common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java @@ -43,6 +43,7 @@ public HiveChar(HiveChar hc, int len) { /** * Set char value, padding or truncating the value to the size of len parameter. */ + @Override public void setValue(String val, int len) { super.setValue(HiveBaseChar.getPaddedValue(val, len), -1); } @@ -59,15 +60,18 @@ public String getPaddedValue() { return value; } + @Override public int getCharacterLength() { String strippedValue = getStrippedValue(); return strippedValue.codePointCount(0, strippedValue.length()); } + @Override public String toString() { return getPaddedValue(); } + @Override public int compareTo(HiveChar rhs) { if (rhs == this) { return 0; @@ -75,6 +79,7 @@ public int compareTo(HiveChar rhs) { return this.getStrippedValue().compareTo(rhs.getStrippedValue()); } + @Override public boolean equals(Object rhs) { if (rhs == this) { return true; @@ -85,7 +90,9 @@ public boolean equals(Object rhs) { return this.getStrippedValue().equals(((HiveChar) rhs).getStrippedValue()); } + @Override public int hashCode() { return getStrippedValue().hashCode(); } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index e7d71595c7a..1a583115308 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -18,13 +18,6 @@ package org.apache.hadoop.hive.ql; -import java.io.FileNotFoundException; -import java.text.MessageFormat; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - import org.antlr.runtime.tree.Tree; import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; @@ -35,6 +28,13 @@ import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.security.AccessControlException; +import java.io.FileNotFoundException; +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + /** * List of all error messages. * This list contains both compile time and run-time errors. @@ -469,6 +469,7 @@ LOAD_DATA_LAUNCH_JOB_PARSE_ERROR(10416, "Encountered parse error while parsing rewritten load data into insert query"), RESOURCE_PLAN_ALREADY_EXISTS(10417, "Resource plan {0} already exists", true), RESOURCE_PLAN_NOT_EXISTS(10418, "Resource plan {0} does not exist", true), + INCOMPATIBLE_STRUCT(10419, "Incompatible structs.", true), //========================== 20000 range starts here ========================// diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java index 04800cca91b..9a525adeb90 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java @@ -20,25 +20,24 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.stream.Collectors; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.rel.AbstractRelNode; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Join; -import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.core.Project; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexUtil; @@ -59,15 +58,18 @@ import com.google.common.collect.Multimaps; import com.google.common.collect.Sets; -public abstract class HivePointLookupOptimizerRule extends RelOptRule { - /** - * This optimization will take a Filter or expression, and if its predicate contains - * an OR operator whose children are constant equality expressions, it will try - * to generate an IN clause (which is more efficient). If the OR operator contains - * AND operator children, the optimization might generate an IN clause that uses - * structs. + * This optimization attempts to identify and close expanded INs. + * + * Basically: + *
+ * (c) IN ( v1, v2, ...) <=> c1=v1 || c1=v2 || ...
+ * 
+ * If c is struct; then c=v1 is a group of anded equations. */ +public abstract class HivePointLookupOptimizerRule extends RelOptRule { + + /** Rule adapter to apply the transformation to Filter conditions. */ public static class FilterCondition extends HivePointLookupOptimizerRule { public FilterCondition (int minNumORClauses) { super(operand(Filter.class, any()), minNumORClauses); @@ -78,22 +80,20 @@ public void onMatch(RelOptRuleCall call) { final Filter filter = call.rel(0); final RexBuilder rexBuilder = filter.getCluster().getRexBuilder(); final RexNode condition = RexUtil.pullFactors(rexBuilder, filter.getCondition()); - analyzeCondition(call , rexBuilder, filter, condition); - } - @Override protected RelNode copyNode(AbstractRelNode node, RexNode newCondition) { - final Filter filter = (Filter) node; - return filter.copy(filter.getTraitSet(), filter.getInput(), newCondition); + RexNode newCondition = analyzeRexNode(rexBuilder, condition); + + // If we could not transform anything, we bail out + if (newCondition.toString().equals(condition.toString())) { + return; + } + RelNode newNode = filter.copy(filter.getTraitSet(), filter.getInput(), newCondition); + + call.transformTo(newNode); } } -/** - * This optimization will take a Join or expression, and if its join condition contains - * an OR operator whose children are constant equality expressions, it will try - * to generate an IN clause (which is more efficient). If the OR operator contains - * AND operator children, the optimization might generate an IN clause that uses - * structs. - */ + /** Rule adapter to apply the transformation to Join conditions. */ public static class JoinCondition extends HivePointLookupOptimizerRule { public JoinCondition (int minNumORClauses) { super(operand(Join.class, any()), minNumORClauses); @@ -104,18 +104,55 @@ public void onMatch(RelOptRuleCall call) { final Join join = call.rel(0); final RexBuilder rexBuilder = join.getCluster().getRexBuilder(); final RexNode condition = RexUtil.pullFactors(rexBuilder, join.getCondition()); - analyzeCondition(call , rexBuilder, join, condition); + + RexNode newCondition = analyzeRexNode(rexBuilder, condition); + + // If we could not transform anything, we bail out + if (newCondition.toString().equals(condition.toString())) { + return; + } + + RelNode newNode = join.copy(join.getTraitSet(), + newCondition, + join.getLeft(), + join.getRight(), + join.getJoinType(), + join.isSemiJoinDone()); + + call.transformTo(newNode); + } + } + + /** Rule adapter to apply the transformation to Projections. */ + public static class ProjectionExpressions extends HivePointLookupOptimizerRule { + public ProjectionExpressions(int minNumORClauses) { + super(operand(Project.class, any()), minNumORClauses); } - @Override protected RelNode copyNode(AbstractRelNode node, RexNode newCondition) { - final Join join = (Join) node; - return join.copy(join.getTraitSet(), - newCondition, - join.getLeft(), - join.getRight(), - join.getJoinType(), - join.isSemiJoinDone()); + @Override + public void onMatch(RelOptRuleCall call) { + final Project project = call.rel(0); + boolean changed = false; + final RexBuilder rexBuilder = project.getCluster().getRexBuilder(); + List newProjects = new ArrayList<>(); + for (RexNode oldNode : project.getProjects()) { + RexNode newNode = analyzeRexNode(rexBuilder, oldNode); + if (!newNode.toString().equals(oldNode.toString())) { + changed = true; + newProjects.add(newNode); + } else { + newProjects.add(oldNode); + } + } + if (!changed) { + return; + } + Project newProject = project.copy(project.getTraitSet(), project.getInput(), newProjects, + project.getRowType(), project.getFlags()); + call.transformTo(newProject); + } + } protected static final Logger LOG = LoggerFactory.getLogger(HivePointLookupOptimizerRule.class); @@ -123,37 +160,21 @@ public void onMatch(RelOptRuleCall call) { // Minimum number of OR clauses needed to transform into IN clauses protected final int minNumORClauses; - protected abstract RelNode copyNode(AbstractRelNode node, RexNode newCondition); - protected HivePointLookupOptimizerRule( RelOptRuleOperand operand, int minNumORClauses) { super(operand); this.minNumORClauses = minNumORClauses; } - public void analyzeCondition(RelOptRuleCall call, - RexBuilder rexBuilder, - AbstractRelNode node, - RexNode condition) { - + public RexNode analyzeRexNode(RexBuilder rexBuilder, RexNode condition) { // 1. We try to transform possible candidates - RexTransformIntoInClause transformIntoInClause = new RexTransformIntoInClause(rexBuilder, node, - minNumORClauses); + RexTransformIntoInClause transformIntoInClause = new RexTransformIntoInClause(rexBuilder, minNumORClauses); RexNode newCondition = transformIntoInClause.apply(condition); // 2. We merge IN expressions RexMergeInClause mergeInClause = new RexMergeInClause(rexBuilder); newCondition = mergeInClause.apply(newCondition); - - // 3. If we could not transform anything, we bail out - if (newCondition.toString().equals(condition.toString())) { - return; - } - - // 4. We create the Filter/Join with the new condition - RelNode newNode = copyNode(node, newCondition); - - call.transformTo(newNode); + return newCondition; } @@ -162,11 +183,9 @@ public void analyzeCondition(RelOptRuleCall call, */ protected static class RexTransformIntoInClause extends RexShuttle { private final RexBuilder rexBuilder; - private final AbstractRelNode nodeOp; private final int minNumORClauses; - RexTransformIntoInClause(RexBuilder rexBuilder, AbstractRelNode nodeOp, int minNumORClauses) { - this.nodeOp = nodeOp; + RexTransformIntoInClause(RexBuilder rexBuilder, int minNumORClauses) { this.rexBuilder = rexBuilder; this.minNumORClauses = minNumORClauses; } @@ -180,7 +199,7 @@ public RexNode visitCall(RexCall inputCall) { case OR: try { RexNode newNode = transformIntoInClauseCondition(rexBuilder, - nodeOp.getRowType(), call, minNumORClauses); + call, minNumORClauses); if (newNode != null) { return newNode; } @@ -196,18 +215,56 @@ public RexNode visitCall(RexCall inputCall) { } /** - * Represents a simple contraint. + * This class just wraps around a RexNode enables equals/hashCode based on toString. + * + * After CALCITE-2632 this might not be needed anymore */ + static class RexNodeRef { + + public static Comparator COMPARATOR = + (RexNodeRef o1, RexNodeRef o2) -> o1.node.toString().compareTo(o2.node.toString()); + private RexNode node; + + public RexNodeRef(RexNode node) { + this.node = node; + } + + public RexNode getRexNode() { + return node; + } + + @Override + public int hashCode() { + return node.toString().hashCode(); + } + + @Override + public boolean equals(Object o) { + if (o instanceof RexNodeRef) { + RexNodeRef otherRef = (RexNodeRef) o; + return node.toString().equals(otherRef.node.toString()); + } + return false; + } + + @Override + public String toString() { + return "ref for:" + node.toString(); + } + } + /** + * Represents a contraint. * * Example: a=1 + * substr(a,1,2) = concat('asd','xxx') */ static class Constraint { - private RexLiteral literal; - private RexInputRef inputRef; + private RexNode exprNode; + private RexNode constNode; - public Constraint(RexInputRef inputRef, RexLiteral literal) { - this.literal = literal; - this.inputRef = inputRef; + public Constraint(RexNode exprNode, RexNode constNode) { + this.exprNode = constNode; + this.constNode = exprNode; } /** @@ -223,21 +280,31 @@ public static Constraint of(RexNode n) { } RexNode opA = call.operands.get(0); RexNode opB = call.operands.get(1); - if (opA instanceof RexLiteral && opB instanceof RexInputRef) { - RexLiteral rexLiteral = (RexLiteral) opA; - RexInputRef rexInputRef = (RexInputRef) opB; - return new Constraint(rexInputRef, rexLiteral); + if (RexUtil.isNull(opA) || RexUtil.isNull(opB)) { + // dont try to compare nulls + return null; + } + if (isConstExpr(opA) && isColumnExpr(opB)) { + return new Constraint(opB, opA); } - if (opA instanceof RexInputRef && opB instanceof RexLiteral) { - RexLiteral rexLiteral = (RexLiteral) opB; - RexInputRef rexInputRef = (RexInputRef) opA; - return new Constraint(rexInputRef, rexLiteral); + if (isColumnExpr(opA) && isConstExpr(opB)) { + return new Constraint(opA, opB); } return null; } - public RexInputRef getKey() { - return inputRef; + private static boolean isColumnExpr(RexNode node) { + return !node.getType().isStruct() && HiveCalciteUtil.getInputRefs(node).size() > 0 + && HiveCalciteUtil.isDeterministic(node); + } + + private static boolean isConstExpr(RexNode node) { + return !node.getType().isStruct() && HiveCalciteUtil.getInputRefs(node).size() == 0 + && HiveCalciteUtil.isDeterministic(node); + } + + public RexNodeRef getKey() { + return new RexNodeRef(constNode); } } @@ -254,17 +321,17 @@ public RexInputRef getKey() { * */ static class ConstraintGroup { + public static final Function> KEY_FUNCTION = + new Function>() { - public static final Function> KEY_FUNCTION = new Function>() { - - @Override - public Set apply(ConstraintGroup a) { - return a.key; - } - }; - private Map constraints = new HashMap<>(); + @Override + public Set apply(ConstraintGroup cg) { + return cg.key; + } + }; + private Map constraints = new HashMap<>(); private RexNode originalRexNode; - private final Set key; + private final Set key; public ConstraintGroup(RexNode rexNode) { originalRexNode = rexNode; @@ -289,21 +356,21 @@ public ConstraintGroup(RexNode rexNode) { key = constraints.keySet(); } - public List getValuesInOrder(List columns) throws SemanticException { + public List getValuesInOrder(List columns) throws SemanticException { List ret = new ArrayList<>(); - for (RexInputRef rexInputRef : columns) { + for (RexNodeRef rexInputRef : columns) { Constraint constraint = constraints.get(rexInputRef); if (constraint == null) { throw new SemanticException("Unable to find constraint which was earlier added."); } - ret.add(constraint.literal); + ret.add(constraint.exprNode); } return ret; } } - private RexNode transformIntoInClauseCondition(RexBuilder rexBuilder, RelDataType inputSchema, - RexNode condition, int minNumORClauses) throws SemanticException { + private RexNode transformIntoInClauseCondition(RexBuilder rexBuilder, RexNode condition, + int minNumORClauses) throws SemanticException { assert condition.getKind() == SqlKind.OR; ImmutableList operands = RexUtil.flattenOr(((RexCall) condition).getOperands()); @@ -318,10 +385,10 @@ private RexNode transformIntoInClauseCondition(RexBuilder rexBuilder, RelDataTyp allNodes.add(m); } - Multimap, ConstraintGroup> assignmentGroups = + Multimap, ConstraintGroup> assignmentGroups = Multimaps.index(allNodes, ConstraintGroup.KEY_FUNCTION); - for (Entry, Collection> sa : assignmentGroups.asMap().entrySet()) { + for (Entry, Collection> sa : assignmentGroups.asMap().entrySet()) { // skip opaque if (sa.getKey().size() == 0) { continue; @@ -351,13 +418,15 @@ private RexNode transformIntoInClauseCondition(RexBuilder rexBuilder, RelDataTyp } - private RexNode buildInFor(Set set, Collection value) throws SemanticException { + private RexNode buildInFor(Set set, Collection value) throws SemanticException { - List columns = new ArrayList(); + List columns = new ArrayList<>(); columns.addAll(set); + columns.sort(RexNodeRef.COMPARATOR); Listoperands = new ArrayList<>(); - operands.add(useStructIfNeeded(columns)); + List columnNodes = columns.stream().map(n -> n.getRexNode()).collect(Collectors.toList()); + operands.add(useStructIfNeeded(columnNodes)); for (ConstraintGroup node : value) { List values = node.getValuesInOrder(columns); operands.add(useStructIfNeeded(values)); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index c9df668d4a7..3a51d9795b0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -910,7 +910,7 @@ public static void validateCheckConstraint(List cols, List stack, NodeProcessorCtx procCtx, } public static ExprNodeConstantDesc createDecimal(String strVal, boolean notNull) { - // Note: the normalize() call with rounding in HiveDecimal will currently reduce the - // precision and scale of the value by throwing away trailing zeroes. This may or may - // not be desirable for the literals; however, this used to be the default behavior - // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now. HiveDecimal hd = HiveDecimal.create(strVal); if (notNull && hd == null) { return null; } + return new ExprNodeConstantDesc(adjustType(hd), hd); + } + + private static DecimalTypeInfo adjustType(HiveDecimal hd) { + // Note: the normalize() call with rounding in HiveDecimal will currently reduce the + // precision and scale of the value by throwing away trailing zeroes. This may or may + // not be desirable for the literals; however, this used to be the default behavior + // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now. int prec = 1; int scale = 0; if (hd != null) { @@ -368,7 +373,7 @@ public static ExprNodeConstantDesc createDecimal(String strVal, boolean notNull) scale = hd.scale(); } DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale); - return new ExprNodeConstantDesc(typeInfo, hd); + return typeInfo; } } @@ -1163,36 +1168,59 @@ protected ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, ExprNodeDesc constChild = children.get(constIdx); ExprNodeDesc columnChild = children.get(1 - constIdx); - final PrimitiveTypeInfo colTypeInfo = - TypeInfoFactory.getPrimitiveTypeInfo(columnChild.getTypeString().toLowerCase()); - ExprNodeDesc newChild = interpretNodeAs(colTypeInfo, constChild); - if (newChild == null) { - // non-interpretabe as that type... - if (genericUDF instanceof GenericUDFOPEqual) { - return new ExprNodeConstantDesc(false); - } - } else { - children.set(constIdx, newChild); + final PrimitiveTypeInfo colTypeInfo = + TypeInfoFactory.getPrimitiveTypeInfo(columnChild.getTypeString().toLowerCase()); + ExprNodeDesc newChild = interpretNodeAs(colTypeInfo, constChild); + if (newChild == null) { + // non-interpretable as target type... + // TODO: all comparisons with null should result in null + if (genericUDF instanceof GenericUDFOPEqual + && !(genericUDF instanceof GenericUDFOPEqualNS)) { + return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, null); } + } else { + children.set(constIdx, newChild); + } } - if (genericUDF instanceof GenericUDFIn && children.get(0) instanceof ExprNodeColumnDesc) { - ExprNodeColumnDesc columnDesc = (ExprNodeColumnDesc) children.get(0); - final PrimitiveTypeInfo colTypeInfo = - TypeInfoFactory.getPrimitiveTypeInfo(columnDesc.getTypeString().toLowerCase()); + if (genericUDF instanceof GenericUDFIn) { + + ExprNodeDesc columnDesc = children.get(0); List outputOpList = children.subList(1, children.size()); ArrayList inOperands = new ArrayList<>(outputOpList); outputOpList.clear(); + boolean hasNullValue = false; for (ExprNodeDesc oldChild : inOperands) { - if(oldChild !=null && oldChild instanceof ExprNodeConstantDesc) { - ExprNodeDesc newChild = interpretNodeAs(colTypeInfo, oldChild); - if(newChild == null) { - // non interpretable as target type; skip - continue; + if (oldChild == null) { + hasNullValue = true; + continue; + } + ExprNodeDesc newChild = interpretNodeAsStruct(columnDesc, oldChild); + if (newChild == null) { + hasNullValue = true; + continue; + } + outputOpList.add(newChild); + } + + if (hasNullValue) { + ExprNodeConstantDesc nullConst = new ExprNodeConstantDesc(columnDesc.getTypeInfo(), null); + if (outputOpList.size() == 0) { + // we have found only null values...remove the IN ; it will be null all the time. + return nullConst; + } + outputOpList.add(nullConst); + } + if (!ctx.isCBOExecuted()) { + ArrayList orOperands = TypeCheckProcFactoryUtils.rewriteInToOR(children); + if (orOperands != null) { + if (orOperands.size() == 1) { + orOperands.add(new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, false)); } - outputOpList.add(newChild); - }else{ - outputOpList.add(oldChild); + funcText = "or"; + genericUDF = new GenericUDFOPOr(); + children.clear(); + children.addAll(orOperands); } } } @@ -1258,48 +1286,145 @@ protected ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, return desc; } - private ExprNodeDesc interpretNodeAs(PrimitiveTypeInfo colTypeInfo, ExprNodeDesc constChild) { + /** + * Interprets the given value as columnDesc if possible + */ + private static ExprNodeDesc interpretNodeAsStruct(ExprNodeDesc columnDesc, ExprNodeDesc valueDesc) + throws SemanticException { + if(columnDesc instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc exprNodeColumnDesc = (ExprNodeColumnDesc) columnDesc; + final PrimitiveTypeInfo typeInfo = + TypeInfoFactory.getPrimitiveTypeInfo(exprNodeColumnDesc.getTypeString().toLowerCase()); + return interpretNodeAs(typeInfo, valueDesc); + } + if (ExprNodeDescUtils.isStructUDF(columnDesc) && ExprNodeDescUtils.isConstantStruct(valueDesc)) { + List columnChilds = ((ExprNodeGenericFuncDesc) columnDesc).getChildren(); + ExprNodeConstantDesc valueConstDesc = (ExprNodeConstantDesc) valueDesc; + StructTypeInfo structTypeInfo = (StructTypeInfo) valueConstDesc.getTypeInfo(); + ArrayList structFieldInfos = structTypeInfo.getAllStructFieldTypeInfos(); + ArrayList newStructFieldInfos = new ArrayList<>(); + + if (columnChilds.size() != structFieldInfos.size()) { + throw new SemanticException(ErrorMsg.INCOMPATIBLE_STRUCT.getMsg(columnChilds + " and " + structFieldInfos)); + } + List oldValues = (List) valueConstDesc.getValue(); + List newValues = new ArrayList<>(); + for (int i = 0; i < columnChilds.size(); i++) { + newStructFieldInfos.add(columnChilds.get(i).getTypeInfo()); + Object newValue = interpretConstantAsPrimitive( + (PrimitiveTypeInfo) columnChilds.get(i).getTypeInfo(), + oldValues.get(i), + structFieldInfos.get(i)); + newValues.add(newValue); + } + StructTypeInfo sti = new StructTypeInfo(); + sti.setAllStructFieldTypeInfos(newStructFieldInfos); + sti.setAllStructFieldNames(structTypeInfo.getAllStructFieldNames()); + return new ExprNodeConstantDesc(sti, newValues); + + } + if (ExprNodeDescUtils.isStructUDF(columnDesc) && ExprNodeDescUtils.isStructUDF(valueDesc)) { + List columnChilds = ((ExprNodeGenericFuncDesc) columnDesc).getChildren(); + List valueChilds = ((ExprNodeGenericFuncDesc) valueDesc).getChildren(); + if (columnChilds.size() != valueChilds.size()) { + throw new SemanticException(ErrorMsg.INCOMPATIBLE_STRUCT.getMsg(columnChilds + " and " + valueChilds)); + } + List oldValueChilds = new ArrayList<>(valueChilds); + valueChilds.clear(); + for (int i = 0; i < oldValueChilds.size(); i++) { + ExprNodeDesc newValue = interpretNodeAsStruct(columnChilds.get(i), oldValueChilds.get(i)); + valueChilds.add(newValue); + } + } + return valueDesc; + } + + private static ExprNodeDesc interpretNodeAs(PrimitiveTypeInfo colTypeInfo, ExprNodeDesc constChild) { if (constChild instanceof ExprNodeConstantDesc) { // Try to narrow type of constant Object constVal = ((ExprNodeConstantDesc) constChild).getValue(); - String constType = constChild.getTypeString().toLowerCase(); - if (constVal instanceof Number || constVal instanceof String) { - try { - PrimitiveTypeEntry primitiveTypeEntry = colTypeInfo.getPrimitiveTypeEntry(); - if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) { - return new ExprNodeConstantDesc(new Integer(constVal.toString())); - } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) { - return new ExprNodeConstantDesc(new Long(constVal.toString())); - } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) { - return new ExprNodeConstantDesc(new Double(constVal.toString())); - } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) { - return new ExprNodeConstantDesc(new Float(constVal.toString())); - } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) { - return new ExprNodeConstantDesc(new Byte(constVal.toString())); - } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) { - return new ExprNodeConstantDesc(new Short(constVal.toString())); - } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) { - return NumExprProcessor.createDecimal(constVal.toString(), false); - } - } catch (NumberFormatException nfe) { - LOG.trace("Failed to narrow type of constant", nfe); - if (!NumberUtils.isNumber(constVal.toString())) { - return null; - } + if (constVal == null) { + // adjust type of null + return new ExprNodeConstantDesc(colTypeInfo, null); + } + Object newConst = interpretConstantAsPrimitive(colTypeInfo, constVal, constChild.getTypeInfo()); + if (newConst == null) { + return null; + } + if(newConst == constVal) { + return constChild; + } else { + return new ExprNodeConstantDesc(adjustType(colTypeInfo, newConst), newConst); + } + } + return constChild; + } + + private static TypeInfo adjustType(PrimitiveTypeInfo colTypeInfo, Object newConst) { + if (newConst instanceof HiveDecimal) { + return NumExprProcessor.adjustType((HiveDecimal) newConst); + } + return colTypeInfo; + } + + private static Object interpretConstantAsPrimitive(PrimitiveTypeInfo colTypeInfo, Object constVal, + TypeInfo constTypeInfo) { + String constTypeInfoName = constTypeInfo.getTypeName(); + if (constVal instanceof Number || constVal instanceof String) { + try { + PrimitiveTypeEntry primitiveTypeEntry = colTypeInfo.getPrimitiveTypeEntry(); + if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) { + return (new Integer(constVal.toString())); + } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) { + return (new Long(constVal.toString())); + } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) { + return (new Double(constVal.toString())); + } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) { + return (new Float(constVal.toString())); + } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) { + return (new Byte(constVal.toString())); + } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) { + return (new Short(constVal.toString())); + } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) { + return HiveDecimal.create(constVal.toString()); + } + } catch (NumberFormatException nfe) { + LOG.trace("Failed to narrow type of constant", nfe); + if (!NumberUtils.isNumber(constVal.toString())) { + return null; } } + } + + // Comparision of decimal and float/double happens in float/double. + if (constVal instanceof HiveDecimal) { + HiveDecimal hiveDecimal = (HiveDecimal) constVal; - // if column type is char and constant type is string, then convert the constant to char - // type with padded spaces. - if (constType.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME) && colTypeInfo instanceof CharTypeInfo) { - final Object originalValue = ((ExprNodeConstantDesc) constChild).getValue(); - final String constValue = originalValue.toString(); - final int length = TypeInfoUtils.getCharacterLengthForType(colTypeInfo); - final HiveChar newValue = new HiveChar(constValue, length); - return new ExprNodeConstantDesc(colTypeInfo, newValue); + PrimitiveTypeEntry primitiveTypeEntry = colTypeInfo.getPrimitiveTypeEntry(); + if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) { + return hiveDecimal.doubleValue(); + } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) { + return hiveDecimal.floatValue(); } + return hiveDecimal; } - return constChild; + + // TODO : Char and string comparison happens in char. But, varchar and string comparison happens in String. + + // if column type is char and constant type is string, then convert the constant to char + // type with padded spaces. + if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME) && colTypeInfo instanceof CharTypeInfo) { + final String constValue = constVal.toString(); + final int length = TypeInfoUtils.getCharacterLengthForType(colTypeInfo); + HiveChar newValue = new HiveChar(constValue, length); + HiveChar maxCharConst = new HiveChar(constValue, HiveChar.MAX_CHAR_LENGTH); + if (maxCharConst.equals(newValue)) { + return newValue; + } else { + return null; + } + } + return constVal; } private boolean canConvertIntoNvl(GenericUDF genericUDF, ArrayList children) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactoryUtils.java ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactoryUtils.java new file mode 100644 index 00000000000..b0544f3d6f5 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactoryUtils.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +import com.google.common.collect.Lists; + +public class TypeCheckProcFactoryUtils { + + static ArrayList rewriteInToOR(ArrayList inOperands) throws SemanticException { + ExprNodeDesc columnDesc = inOperands.get(0); + + ArrayList orOperands = new ArrayList<>(); + for (int i = 1; i < inOperands.size(); i++) { + ExprNodeDesc andExpr = buildEqualsArr(columnDesc, inOperands.get(i)); + if (andExpr == null) { + return null; + } + orOperands.add(andExpr); + } + return orOperands; + } + + private static ExprNodeDesc buildEqualsArr(ExprNodeDesc columnDesc, ExprNodeDesc exprNodeDesc) + throws SemanticException { + List lNodes = asListOfNodes(columnDesc); + List rNodes = asListOfNodes(exprNodeDesc); + if (lNodes == null || rNodes == null) { + // something went wrong + return null; + } + if (lNodes.size() != rNodes.size()) { + throw new SemanticException(ErrorMsg.INCOMPATIBLE_STRUCT.getMsg(columnDesc + " and " + exprNodeDesc)); + } + + List ret = new ArrayList<>(); + for (int i = 0; i < lNodes.size(); i++) { + ret.add(buildEquals(lNodes.get(i), rNodes.get(i))); + } + return buildAnd(ret); + } + + private static ExprNodeGenericFuncDesc buildEquals(ExprNodeDesc columnDesc, ExprNodeDesc valueDesc) { + return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPEqual(), "=", + Lists.newArrayList(columnDesc, valueDesc)); + } + + private static ExprNodeDesc buildAnd(List values) { + if (values.size() == 1) { + return values.get(0); + } else { + return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPAnd(), "and", values); + } + } + + private static List asListOfNodes(ExprNodeDesc desc) { + ExprNodeDesc valueDesc = desc; + if (ExprNodeDescUtils.isStructUDF(desc)) { + List valueChilds = ((ExprNodeGenericFuncDesc) valueDesc).getChildren(); + for (ExprNodeDesc exprNodeDesc : valueChilds) { + if (!isSafeExpression(exprNodeDesc)) { + return null; + } + } + return valueChilds; + } + if (ExprNodeDescUtils.isConstantStruct(valueDesc)) { + ExprNodeConstantDesc valueConstDesc = (ExprNodeConstantDesc) valueDesc; + List oldValues = (List) valueConstDesc.getValue(); + StructTypeInfo structTypeInfo = (StructTypeInfo) valueConstDesc.getTypeInfo(); + ArrayList structFieldInfos = structTypeInfo.getAllStructFieldTypeInfos(); + + List ret = new ArrayList<>(); + for (int i = 0; i < oldValues.size(); i++) { + ret.add(new ExprNodeConstantDesc(structFieldInfos.get(i), oldValues.get(i))); + } + return ret; + } + if (isSafeExpression(desc)) { + return Lists.newArrayList(desc); + } + + return null; + } + + private static boolean isSafeExpression(ExprNodeDesc desc) { + TypeInfo typeInfo = desc.getTypeInfo(); + if (typeInfo.getCategory() != Category.PRIMITIVE) { + return false; + } + if (isConstantOrColumn(desc)) { + return true; + } + if (desc instanceof ExprNodeGenericFuncDesc) { + ExprNodeGenericFuncDesc exprNodeGenericFuncDesc = (ExprNodeGenericFuncDesc) desc; + if (FunctionRegistry.isConsistentWithinQuery(exprNodeGenericFuncDesc.getGenericUDF())) { + for (ExprNodeDesc child : exprNodeGenericFuncDesc.getChildren()) { + if (!isSafeExpression(child)) { + return false; + } + } + return true; + } + } + return false; + } + + private static boolean isConstantOrColumn(ExprNodeDesc desc) { + return desc instanceof ExprNodeColumnDesc || desc instanceof ExprNodeConstantDesc; + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java index c274fd7cc9b..596640ee49d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java @@ -18,18 +18,14 @@ package org.apache.hadoop.hive.ql.plan; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.ql.exec.RowSchema; +import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; @@ -38,6 +34,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -45,10 +42,15 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.util.ReflectionUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + public class ExprNodeDescUtils { @@ -571,7 +573,7 @@ public static void getExprNodeColumnDesc(List exprDescList, /** * Get Map of ExprNodeColumnDesc HashCode to ExprNodeColumnDesc. - * + * * @param exprDesc * @param hashCodeToColumnDescMap * Assumption: If two ExprNodeColumnDesc have same hash code then @@ -642,7 +644,9 @@ public static PrimitiveTypeInfo deriveMinArgumentCast( // We only do the minimum cast for decimals. Other types are assumed safe; fix if needed. // We also don't do anything for non-primitive children (maybe we should assert). if ((pti.getPrimitiveCategory() != PrimitiveCategory.DECIMAL) - || (!(childExpr.getTypeInfo() instanceof PrimitiveTypeInfo))) return pti; + || (!(childExpr.getTypeInfo() instanceof PrimitiveTypeInfo))) { + return pti; + } PrimitiveTypeInfo childTi = (PrimitiveTypeInfo)childExpr.getTypeInfo(); // If the child is also decimal, no cast is needed (we hope - can target type be narrower?). return HiveDecimalUtils.getDecimalTypeForPrimitiveCategory(childTi); @@ -652,7 +656,7 @@ public static PrimitiveTypeInfo deriveMinArgumentCast( * Build ExprNodeColumnDesc for the projections in the input operator from * sartpos to endpos(both included). Operator must have an associated * colExprMap. - * + * * @param inputOp * Input Hive Operator * @param startPos @@ -684,7 +688,7 @@ public static PrimitiveTypeInfo deriveMinArgumentCast( } return exprColLst; - } + } public static List flattenExprList(List sourceList) { ArrayList result = new ArrayList(sourceList.size()); @@ -998,4 +1002,17 @@ public static boolean isIntegerType(ExprNodeDesc expr) { } return false; } + + public static boolean isConstantStruct(ExprNodeDesc valueDesc) { + return valueDesc instanceof ExprNodeConstantDesc && valueDesc.getTypeInfo() instanceof StructTypeInfo; + } + + public static boolean isStructUDF(ExprNodeDesc columnDesc) { + if (columnDesc instanceof ExprNodeGenericFuncDesc) { + ExprNodeGenericFuncDesc exprNodeGenericFuncDesc = (ExprNodeGenericFuncDesc) columnDesc; + return (exprNodeGenericFuncDesc.getGenericUDF() instanceof GenericUDFStruct); + } + return false; + } + } diff --git ql/src/test/queries/clientnegative/udf_in_2.q ql/src/test/queries/clientnegative/udf_in_2.q deleted file mode 100644 index 2288054d583..00000000000 --- ql/src/test/queries/clientnegative/udf_in_2.q +++ /dev/null @@ -1,2 +0,0 @@ -select 1=1 in (true, false); - diff --git ql/src/test/queries/clientpositive/acid_no_buckets.q ql/src/test/queries/clientpositive/acid_no_buckets.q index 552010a63b9..48382fd9652 100644 --- ql/src/test/queries/clientpositive/acid_no_buckets.q +++ ql/src/test/queries/clientpositive/acid_no_buckets.q @@ -17,219 +17,10 @@ CREATE TABLE srcpart_acid (key STRING, value STRING) PARTITIONED BY (ds STRING, insert into srcpart_acid PARTITION (ds, hr) select * from srcpart; --2 rows for 413, 1 row for 43, 2 for 213, 1 for 44 in kv1.txt (in each partition) -select ds, hr, key, value from srcpart_acid where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer); analyze table srcpart_acid PARTITION(ds, hr) compute statistics; analyze table srcpart_acid PARTITION(ds, hr) compute statistics for columns; explain update srcpart_acid set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'; update srcpart_acid set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'; -select ds, hr, key, value from srcpart_acid where value like '%updated' order by ds, hr, cast(key as integer); -insert into srcpart_acid PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003'); -select ds, hr, key, value from srcpart_acid where cast(key as integer) > 1000 order by ds, hr, cast(key as integer); -describe formatted srcpart_acid; -describe formatted srcpart_acid key; - -analyze table srcpart_acid PARTITION(ds, hr) compute statistics; -analyze table srcpart_acid PARTITION(ds, hr) compute statistics for columns; - --- make sure the stats stay the same after analyze (insert and update above also update stats) -describe formatted srcpart_acid; -describe formatted srcpart_acid key; - -explain delete from srcpart_acid where key in( '1001', '213', '43'); ---delete some rows from initial load, some that were updated and some that were inserted -delete from srcpart_acid where key in( '1001', '213', '43'); - ---make sure we deleted everything that should've been deleted -select count(*) from srcpart_acid where key in( '1001', '213', '43'); ---make sure nothing extra was deleted (2000 + 3 (insert) - 4 - 1 - 8 = 1990) -select count(*) from srcpart_acid; - ---todo: should really have a way to run compactor here.... - ---update should match 1 rows in 1 partition ---delete should drop everything from 1 partition ---insert should do nothing -merge into srcpart_acid t using (select distinct ds, hr, key, value from srcpart_acid) s -on s.ds=t.ds and s.hr=t.hr and s.key=t.key and s.value=t.value -when matched and s.ds='2008-04-08' and s.hr=='11' and s.key='44' then update set value=concat(s.value,'updated by merge') -when matched and s.ds='2008-04-08' and s.hr=='12' then delete -when not matched then insert values('this','should','not','be there'); - ---check results ---should be 0 -select count(*) from srcpart_acid where ds='2008-04-08' and hr=='12'; ---should be 1 rows -select ds, hr, key, value from srcpart_acid where value like '%updated by merge'; ---should be 0 -select count(*) from srcpart_acid where ds = 'this' and hr = 'should' and key = 'not' and value = 'be there'; -drop table if exists srcpart_acid; - - -drop table if exists srcpart_acidb; -CREATE TABLE srcpart_acidb (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) CLUSTERED BY(key) INTO 2 BUCKETS stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default'); -insert into srcpart_acidb PARTITION (ds, hr) select * from srcpart; - ---2 rows for 413, 1 row for 43, 2 for 213, 2 for 12 in kv1.txt (in each partition) -select ds, hr, key, value from srcpart_acidb where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer); - -analyze table srcpart_acidb PARTITION(ds, hr) compute statistics; -analyze table srcpart_acidb PARTITION(ds, hr) compute statistics for columns; -explain update srcpart_acidb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'; -update srcpart_acidb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'; -select ds, hr, key, value from srcpart_acidb where value like '%updated' order by ds, hr, cast(key as integer); - -insert into srcpart_acidb PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003'); -select ds, hr, key, value from srcpart_acidb where cast(key as integer) > 1000 order by ds, hr, cast(key as integer); - -analyze table srcpart_acidb PARTITION(ds, hr) compute statistics; -analyze table srcpart_acidb PARTITION(ds, hr) compute statistics for columns; -explain delete from srcpart_acidb where key in( '1001', '213', '43'); ---delete some rows from initial load, some that were updated and some that were inserted -delete from srcpart_acidb where key in( '1001', '213', '43'); - ---make sure we deleted everything that should've been deleted -select count(*) from srcpart_acidb where key in( '1001', '213', '43'); ---make sure nothing extra was deleted (2000 + 3 (insert) - 4 - 1 - 8 = 1990) -select count(*) from srcpart_acidb; - - ---todo: should really have a way to run compactor here.... - ---update should match 1 rows in 1 partition ---delete should drop everything from 1 partition ---insert should do nothing -merge into srcpart_acidb t using (select distinct ds, hr, key, value from srcpart_acidb) s -on s.ds=t.ds and s.hr=t.hr and s.key=t.key and s.value=t.value -when matched and s.ds='2008-04-08' and s.hr=='11' and s.key='44' then update set value=concat(s.value,'updated by merge') -when matched and s.ds='2008-04-08' and s.hr=='12' then delete -when not matched then insert values('this','should','not','be there'); - ---check results ---should be 0 -select count(*) from srcpart_acidb where ds='2008-04-08' and hr=='12'; ---should be 1 rows -select ds, hr, key, value from srcpart_acidb where value like '%updated by merge'; ---should be 0 -select count(*) from srcpart_acidb where ds = 'this' and hr = 'should' and key = 'not' and value = 'be there'; -drop table if exists srcpart_acidb; - - - ---now same thing but vectorized -set hive.vectorized.execution.enabled=true; - -drop table if exists srcpart_acidv; -CREATE TABLE srcpart_acidv (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default'); -insert into srcpart_acidv PARTITION (ds, hr) select * from srcpart; - ---2 rows for 413, 21 row for 43, 2 for 213, 2 for 12 in kv1.txt (in each partition) -select ds, hr, key, value from srcpart_acidv where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer); - -analyze table srcpart_acidv PARTITION(ds, hr) compute statistics; -analyze table srcpart_acidv PARTITION(ds, hr) compute statistics for columns; -explain vectorization only detail -update srcpart_acidv set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'; -update srcpart_acidv set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'; -select ds, hr, key, value from srcpart_acidv where value like '%updated' order by ds, hr, cast(key as integer); - -insert into srcpart_acidv PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003'); -select ds, hr, key, value from srcpart_acidv where cast(key as integer) > 1000 order by ds, hr, cast(key as integer); - -analyze table srcpart_acidv PARTITION(ds, hr) compute statistics; -analyze table srcpart_acidv PARTITION(ds, hr) compute statistics for columns; -explain vectorization only detail -delete from srcpart_acidv where key in( '1001', '213', '43'); ---delete some rows from initial load, some that were updated and some that were inserted -delete from srcpart_acidv where key in( '1001', '213', '43'); - ---make sure we deleted everything that should've been deleted -select count(*) from srcpart_acidv where key in( '1001', '213', '43'); ---make sure nothing extra was deleted (2000 + 3 - 4 - 1 - 8 = 1990) -select count(*) from srcpart_acidv; - ---todo: should really have a way to run compactor here.... - ---update should match 1 rows in 1 partition ---delete should drop everything from 1 partition ---insert should do nothing -explain vectorization only detail -merge into srcpart_acidv t using (select distinct ds, hr, key, value from srcpart_acidv) s -on s.ds=t.ds and s.hr=t.hr and s.key=t.key and s.value=t.value -when matched and s.ds='2008-04-08' and s.hr=='11' and s.key='44' then update set value=concat(s.value,'updated by merge') -when matched and s.ds='2008-04-08' and s.hr=='12' then delete -when not matched then insert values('this','should','not','be there'); -merge into srcpart_acidv t using (select distinct ds, hr, key, value from srcpart_acidv) s -on s.ds=t.ds and s.hr=t.hr and s.key=t.key and s.value=t.value -when matched and s.ds='2008-04-08' and s.hr=='11' and s.key='44' then update set value=concat(s.value,'updated by merge') -when matched and s.ds='2008-04-08' and s.hr=='12' then delete -when not matched then insert values('this','should','not','be there'); - ---check results ---should be 0 -select count(*) from srcpart_acidv where ds='2008-04-08' and hr=='12'; ---should be 1 rows -select ds, hr, key, value from srcpart_acidv where value like '%updated by merge'; ---should be 0 -select count(*) from srcpart_acidv where ds = 'this' and hr = 'should' and key = 'not' and value = 'be there'; -drop table if exists srcpart_acidv; - - - -drop table if exists srcpart_acidvb; -CREATE TABLE srcpart_acidvb (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) CLUSTERED BY(key) INTO 2 BUCKETS stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default'); -insert into srcpart_acidvb PARTITION (ds, hr) select * from srcpart; - ---2 rows for 413, 1 row for 43, 2 for 213, 2 for 12 in kv1.txt (in each partition) -select ds, hr, key, value from srcpart_acidvb where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer); - -analyze table srcpart_acidvb PARTITION(ds, hr) compute statistics; -analyze table srcpart_acidvb PARTITION(ds, hr) compute statistics for columns; -explain vectorization only detail -update srcpart_acidvb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'; -update srcpart_acidvb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'; -select ds, hr, key, value from srcpart_acidvb where value like '%updated' order by ds, hr, cast(key as integer); - -insert into srcpart_acidvb PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003'); -select ds, hr, key, value from srcpart_acidvb where cast(key as integer) > 1000 order by ds, hr, cast(key as integer); - -analyze table srcpart_acidvb PARTITION(ds, hr) compute statistics; -analyze table srcpart_acidvb PARTITION(ds, hr) compute statistics for columns; -explain vectorization only detail -delete from srcpart_acidvb where key in( '1001', '213', '43'); ---delete some rows from initial load, some that were updated and some that were inserted -delete from srcpart_acidvb where key in( '1001', '213', '43'); - ---make sure we deleted everything that should've been deleted -select count(*) from srcpart_acidvb where key in( '1001', '213', '43'); ---make sure nothing extra was deleted (2000 + 3 (insert) - 4 - 1 - 8 = 1990) -select count(*) from srcpart_acidvb; - - ---todo: should really have a way to run compactor here.... - ---update should match 1 rows in 1 partition ---delete should drop everything from 1 partition ---insert should do nothing -explain vectorization only detail -merge into srcpart_acidvb t using (select distinct ds, hr, key, value from srcpart_acidvb) s -on s.ds=t.ds and s.hr=t.hr and s.key=t.key and s.value=t.value -when matched and s.ds='2008-04-08' and s.hr=='11' and s.key='44' then update set value=concat(s.value,'updated by merge') -when matched and s.ds='2008-04-08' and s.hr=='12' then delete -when not matched then insert values('this','should','not','be there'); -merge into srcpart_acidvb t using (select distinct ds, hr, key, value from srcpart_acidvb) s -on s.ds=t.ds and s.hr=t.hr and s.key=t.key and s.value=t.value -when matched and s.ds='2008-04-08' and s.hr=='11' and s.key='44' then update set value=concat(s.value,'updated by merge') -when matched and s.ds='2008-04-08' and s.hr=='12' then delete -when not matched then insert values('this','should','not','be there'); - ---check results ---should be 0 -select count(*) from srcpart_acidvb where ds='2008-04-08' and hr=='12'; ---should be 1 rows -select ds, hr, key, value from srcpart_acidvb where value like '%updated by merge'; ---should be 0 -select count(*) from srcpart_acidvb where ds = 'this' and hr = 'should' and key = 'not' and value = 'be there'; -drop table if exists srcpart_acidvb; diff --git ql/src/test/queries/clientpositive/groupby_multi_single_reducer3.q ql/src/test/queries/clientpositive/groupby_multi_single_reducer3.q index 94aea35c422..03af1cced71 100644 --- ql/src/test/queries/clientpositive/groupby_multi_single_reducer3.q +++ ql/src/test/queries/clientpositive/groupby_multi_single_reducer3.q @@ -3,6 +3,8 @@ -- SORT_QUERY_RESULTS +set hive.cbo.enable=true; + create table e1_n1 (key string, count int); create table e2_n2 (key string, count int); diff --git ql/src/test/queries/clientpositive/in_typecheck_char.q ql/src/test/queries/clientpositive/in_typecheck_char.q new file mode 100644 index 00000000000..3955c4be142 --- /dev/null +++ ql/src/test/queries/clientpositive/in_typecheck_char.q @@ -0,0 +1,24 @@ + +create table ax(s char(1),t char(10)); + +insert into ax values ('a','a'),('a','a '),('b','bb'); + +explain +select 'expected 2',count(*) from ax where s = 'a' and t = 'a'; +select 'expected 2',count(*) from ax where s = 'a' and t = 'a'; + +explain +select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb')); +select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb')); + +select 'expected 2',count(*) from ax where t = 'a '; +select 'expected 2',count(*) from ax where t = 'a '; +select 'expected 0',count(*) from ax where t = 'a d'; + + +select 'expected 2',count(*) from ax where (s,t) in (('a','a'),(null, 'bb')); + + +-- this is right now broken; HIVE-20779 should fix it +explain select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null; +select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null; diff --git ql/src/test/queries/clientpositive/in_typecheck_mixed.q ql/src/test/queries/clientpositive/in_typecheck_mixed.q new file mode 100644 index 00000000000..26c486bf529 --- /dev/null +++ ql/src/test/queries/clientpositive/in_typecheck_mixed.q @@ -0,0 +1,12 @@ +create table t (a string); + +insert into t values ('1'),('x'),('2.0'); + +explain +select * from t where a in (1.0,'x',2); +select * from t where a in (1.0,'x',2); + + +create table ax(s char(1),t char(10)); +insert into ax values ('a','a'),('a','a '),('b','bb'); +select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null; diff --git ql/src/test/queries/clientpositive/in_typecheck_pointlook.q ql/src/test/queries/clientpositive/in_typecheck_pointlook.q new file mode 100644 index 00000000000..c788506f45a --- /dev/null +++ ql/src/test/queries/clientpositive/in_typecheck_pointlook.q @@ -0,0 +1,35 @@ + +create table customer_demographics (cd_marital_status char(1), cd_education_status char(20)); + +insert into customer_demographics values +('M','Unknown'), +('W','Advanced Degree'), +('W','Advanced Degree '), +('W',' Advanced Degree') +; + +set hive.optimize.point.lookup.min=32; + +explain +select count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')); + +select '3 is expected:',count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')); + +set hive.optimize.point.lookup.min=2; + +explain +select count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')); + +select '3 is expected:',count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')); diff --git ql/src/test/queries/clientpositive/in_typecheck_varchar.q ql/src/test/queries/clientpositive/in_typecheck_varchar.q new file mode 100644 index 00000000000..e8dbf46bec6 --- /dev/null +++ ql/src/test/queries/clientpositive/in_typecheck_varchar.q @@ -0,0 +1,16 @@ + +create table ax(s varchar(1),t varchar(10)); + +insert into ax values ('a','a'),('a','a '),('b','bb'); + +explain +select 'expected 1',count(*) from ax where s = 'a' and t = 'a'; +select 'expected 1',count(*) from ax where s = 'a' and t = 'a'; + +explain +select 'expected 2',count(*) from ax where (s,t) in (('a','a'),('b','bb')); +select 'expected 2',count(*) from ax where (s,t) in (('a','a'),('b','bb')); + +select 'expected 0',count(*) from ax where t = 'a '; +select 'expected 0',count(*) from ax where t = 'a '; +select 'expected 0',count(*) from ax where t = 'a d'; diff --git ql/src/test/queries/clientpositive/join45X.q ql/src/test/queries/clientpositive/join45X.q new file mode 100644 index 00000000000..8291c9e3d82 --- /dev/null +++ ql/src/test/queries/clientpositive/join45X.q @@ -0,0 +1,32 @@ +--! qt:dataset:src +--! qt:dataset:src1 + +set hive.strict.checks.cartesian.product=false; + +-- Function with multiple inputs on one side +EXPLAIN +SELECT * +FROM src1 JOIN src +ON (src1.key= 100 and src.key=100) +LIMIT 10; + + +-- Function with multiple inputs on one side +EXPLAIN +SELECT * +FROM src1 JOIN src +ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) +LIMIT 10; + +SELECT * +FROM src1 JOIN src +ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) +LIMIT 10; + + +EXPLAIN + SELECT * + FROM src1 JOIN src + ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) + LIMIT 10 + diff --git ql/src/test/queries/clientpositive/pcs.q ql/src/test/queries/clientpositive/pcs.q index 2e7eff472b5..31a5f0bc055 100644 --- ql/src/test/queries/clientpositive/pcs.q +++ ql/src/test/queries/clientpositive/pcs.q @@ -17,7 +17,6 @@ set hive.optimize.point.lookup.min = 1; explain extended select key, value, ds from pcs_t1 where (ds='2000-04-08' and key=1) or (ds='2000-04-09' and key=2) order by key, value, ds; select key, value, ds from pcs_t1 where (ds='2000-04-08' and key=1) or (ds='2000-04-09' and key=2) order by key, value, ds; -set hive.optimize.point.lookup = false; set hive.optimize.partition.columns.separate=true; set hive.optimize.ppd=true; @@ -27,7 +26,6 @@ select ds from pcs_t1 where struct(ds, key) in (struct('2000-04-08',1), struct(' explain extended select ds from pcs_t1 where struct(ds, key+2) in (struct('2000-04-08',3), struct('2000-04-09',4)); select ds from pcs_t1 where struct(ds, key+2) in (struct('2000-04-08',3), struct('2000-04-09',4)); -set hive.cbo.enable=false; explain extended select /*+ MAPJOIN(pcs_t1) */ a.ds, b.key from pcs_t1 a join pcs_t1 b on a.ds=b.ds where struct(a.ds, a.key, b.ds) in (struct('2000-04-08',1, '2000-04-09'), struct('2000-04-09',2, '2000-04-08')); select /*+ MAPJOIN(pcs_t1) */ a.ds, b.key from pcs_t1 a join pcs_t1 b on a.ds=b.ds where struct(a.ds, a.key, b.ds) in (struct('2000-04-08',1, '2000-04-09'), struct('2000-04-09',2, '2000-04-08')); diff --git ql/src/test/queries/clientpositive/vector_struct_in.q ql/src/test/queries/clientpositive/vector_struct_in.q index 49a9374bd13..ed03214d5a1 100644 --- ql/src/test/queries/clientpositive/vector_struct_in.q +++ ql/src/test/queries/clientpositive/vector_struct_in.q @@ -1,4 +1,4 @@ -set hive.cbo.enable=false; +set hive.cbo.enable=true; set hive.explain.user=false; set hive.tez.dynamic.partition.pruning=false; set hive.vectorized.execution.enabled=true; diff --git ql/src/test/results/clientnegative/udf_in.q.out ql/src/test/results/clientnegative/udf_in.q.out index c032ee42f92..ab55820f755 100644 --- ql/src/test/results/clientnegative/udf_in.q.out +++ ql/src/test/results/clientnegative/udf_in.q.out @@ -1 +1 @@ -FAILED: SemanticException Line 0:-1 Wrong arguments '3': The arguments for IN should be the same type! Types are: {int IN (array)} +FAILED: UDFArgumentTypeException The 2nd argument of EQUAL is expected to a primitive type, but list is found diff --git ql/src/test/results/clientnegative/udf_in_2.q.out ql/src/test/results/clientnegative/udf_in_2.q.out deleted file mode 100644 index 83883e3f9e3..00000000000 --- ql/src/test/results/clientnegative/udf_in_2.q.out +++ /dev/null @@ -1 +0,0 @@ -FAILED: SemanticException Line 0:-1 Wrong arguments 'false': The arguments for IN should be the same type! Types are: {int IN (boolean, boolean)} diff --git ql/src/test/results/clientpositive/alter_partition_coltype.q.out ql/src/test/results/clientpositive/alter_partition_coltype.q.out index f6c3c5642ea..d484f9e2237 100644 --- ql/src/test/results/clientpositive/alter_partition_coltype.q.out +++ ql/src/test/results/clientpositive/alter_partition_coltype.q.out @@ -160,7 +160,7 @@ POSTHOOK: Input: default@alter_coltype #### A masked pattern was here #### OPTIMIZED SQL: SELECT COUNT(*) AS `$f0` FROM `default`.`alter_coltype` -WHERE `ts` = 3 AND `dt` = 100 +WHERE `ts` = 3.0 AND `dt` = 100 STAGE DEPENDENCIES: Stage-0 is a root stage diff --git ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out index a8b595a6590..b6e90777f9d 100644 --- ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out +++ ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out @@ -57,10 +57,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src - filterExpr: (((value) IN ('val_400', 'val_500') and (key) IN (400, 450)) or ((value) IN ('val_100', 'val_200', 'val_300') and (key) IN (100, 150, 200))) (type: boolean) + filterExpr: ((((value = 'val_400') or (value = 'val_500')) and ((key = 400) or (key = 450))) or (((value = 'val_100') or (value = 'val_200') or (value = 'val_300')) and ((key = 100) or (key = 150) or (key = 200)))) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((value) IN ('val_100', 'val_200', 'val_300') and (key) IN (100, 150, 200)) or ((value) IN ('val_400', 'val_500') and (key) IN (400, 450))) (type: boolean) + predicate: ((((value = 'val_100') or (value = 'val_200') or (value = 'val_300')) and ((key = 100) or (key = 150) or (key = 200))) or (((value = 'val_400') or (value = 'val_500')) and ((key = 400) or (key = 450)))) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -68,11 +68,12 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) + Execution mode: vectorized Reduce Operator Tree: Forward Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((KEY._col0) IN (100, 150, 200) and (VALUE._col0) IN ('val_100', 'val_200', 'val_300')) (type: boolean) + predicate: (((KEY._col0 = 100) or (KEY._col0 = 150) or (KEY._col0 = 200)) and ((VALUE._col0 = 'val_100') or (VALUE._col0 = 'val_200') or (VALUE._col0 = 'val_300'))) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -108,7 +109,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Filter Operator - predicate: ((KEY._col0) IN (400, 450) and (VALUE._col0) IN ('val_400', 'val_500')) (type: boolean) + predicate: (((KEY._col0 = 400) or (KEY._col0 = 450)) and ((VALUE._col0 = 'val_400') or (VALUE._col0 = 'val_500'))) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -582,7 +583,7 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((value) IN ('val_100', 'val_200', 'val_300') and (key) IN (100, 150, 200)) or ((value) IN ('val_400', 'val_500') and (key) IN (400, 450))) (type: boolean) + predicate: ((((value = 'val_100') or (value = 'val_200') or (value = 'val_300')) and ((key = 100) or (key = 150) or (key = 200))) or (((value = 'val_400') or (value = 'val_500')) and ((key = 400) or (key = 450)))) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -590,11 +591,12 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) + Execution mode: vectorized Reduce Operator Tree: Forward Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((KEY._col0) IN (100, 150, 200) and (VALUE._col0) IN ('val_100', 'val_200', 'val_300')) (type: boolean) + predicate: (((KEY._col0 = 100) or (KEY._col0 = 150) or (KEY._col0 = 200)) and ((VALUE._col0 = 'val_100') or (VALUE._col0 = 'val_200') or (VALUE._col0 = 'val_300'))) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -630,7 +632,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Filter Operator - predicate: ((KEY._col0) IN (400, 450) and (VALUE._col0) IN ('val_400', 'val_500')) (type: boolean) + predicate: (((KEY._col0 = 400) or (KEY._col0 = 450)) and ((VALUE._col0 = 'val_400') or (VALUE._col0 = 'val_500'))) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() diff --git ql/src/test/results/clientpositive/in_typecheck_char.q.out ql/src/test/results/clientpositive/in_typecheck_char.q.out new file mode 100644 index 00000000000..6948719881a --- /dev/null +++ ql/src/test/results/clientpositive/in_typecheck_char.q.out @@ -0,0 +1,261 @@ +PREHOOK: query: create table ax(s char(1),t char(10)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ax +POSTHOOK: query: create table ax(s char(1),t char(10)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ax +PREHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ax +POSTHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ax +POSTHOOK: Lineage: ax.s SCRIPT [] +POSTHOOK: Lineage: ax.t SCRIPT [] +PREHOOK: query: explain +select 'expected 2',count(*) from ax where s = 'a' and t = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: explain +select 'expected 2',count(*) from ax where s = 'a' and t = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: ax + filterExpr: ((s = 'a') and (t = 'a ')) (type: boolean) + Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((s = 'a') and (t = 'a ')) (type: boolean) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 'expected 2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select 'expected 2',count(*) from ax where s = 'a' and t = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 2',count(*) from ax where s = 'a' and t = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 2 2 +PREHOOK: query: explain +select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb')) +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: explain +select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: ax + filterExpr: (struct(s,t)) IN (const struct('a','a '), const struct('b','bb ')) (type: boolean) + Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (struct(s,t)) IN (const struct('a','a '), const struct('b','bb ')) (type: boolean) + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 'expected 3' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb')) +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 3 3 +PREHOOK: query: select 'expected 2',count(*) from ax where t = 'a ' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 2',count(*) from ax where t = 'a ' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 2 2 +PREHOOK: query: select 'expected 2',count(*) from ax where t = 'a ' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 2',count(*) from ax where t = 'a ' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 2 2 +PREHOOK: query: select 'expected 0',count(*) from ax where t = 'a d' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 0',count(*) from ax where t = 'a d' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 0 0 +PREHOOK: query: select 'expected 2',count(*) from ax where (s,t) in (('a','a'),(null, 'bb')) +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 2',count(*) from ax where (s,t) in (('a','a'),(null, 'bb')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 2 2 +PREHOOK: query: explain select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: explain select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: ax + filterExpr: (((s = 'a') and (t = 'a ')) or ((s = null) and (t = 'bb '))) is null (type: boolean) + Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((s = 'a') and (t = 'a ')) or ((s = null) and (t = 'bb '))) is null (type: boolean) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 'expected 1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 1 1 diff --git ql/src/test/results/clientpositive/in_typecheck_mixed.q.out ql/src/test/results/clientpositive/in_typecheck_mixed.q.out new file mode 100644 index 00000000000..fb3f3304a63 --- /dev/null +++ ql/src/test/results/clientpositive/in_typecheck_mixed.q.out @@ -0,0 +1,99 @@ +PREHOOK: query: create table t (a string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: create table t (a string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +PREHOOK: query: insert into t values ('1'),('x'),('2.0') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@t +POSTHOOK: query: insert into t values ('1'),('x'),('2.0') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@t +POSTHOOK: Lineage: t.a SCRIPT [] +PREHOOK: query: explain +select * from t where a in (1.0,'x',2) +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: explain +select * from t where a in (1.0,'x',2) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: t + filterExpr: ((UDFToDouble(a)) IN (1.0D, 2.0D) or (a = 'x')) (type: boolean) + Statistics: Num rows: 3 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(a)) IN (1.0D, 2.0D) or (a = 'x')) (type: boolean) + Statistics: Num rows: 2 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 3 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 3 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from t where a in (1.0,'x',2) +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select * from t where a in (1.0,'x',2) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +1 +x +2.0 +PREHOOK: query: create table ax(s char(1),t char(10)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ax +POSTHOOK: query: create table ax(s char(1),t char(10)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ax +PREHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ax +POSTHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ax +POSTHOOK: Lineage: ax.s SCRIPT [] +POSTHOOK: Lineage: ax.t SCRIPT [] +PREHOOK: query: select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 1 1 diff --git ql/src/test/results/clientpositive/in_typecheck_pointlook.q.out ql/src/test/results/clientpositive/in_typecheck_pointlook.q.out new file mode 100644 index 00000000000..7a913fec97c --- /dev/null +++ ql/src/test/results/clientpositive/in_typecheck_pointlook.q.out @@ -0,0 +1,181 @@ +PREHOOK: query: create table customer_demographics (cd_marital_status char(1), cd_education_status char(20)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@customer_demographics +POSTHOOK: query: create table customer_demographics (cd_marital_status char(1), cd_education_status char(20)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@customer_demographics +PREHOOK: query: insert into customer_demographics values +('M','Unknown'), +('W','Advanced Degree'), +('W','Advanced Degree '), +('W',' Advanced Degree') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@customer_demographics +POSTHOOK: query: insert into customer_demographics values +('M','Unknown'), +('W','Advanced Degree'), +('W','Advanced Degree '), +('W',' Advanced Degree') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@customer_demographics +POSTHOOK: Lineage: customer_demographics.cd_education_status SCRIPT [] +POSTHOOK: Lineage: customer_demographics.cd_marital_status SCRIPT [] +PREHOOK: query: explain +select count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +PREHOOK: type: QUERY +PREHOOK: Input: default@customer_demographics +#### A masked pattern was here #### +POSTHOOK: query: explain +select count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@customer_demographics +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: customer_demographics + filterExpr: (((cd_marital_status = 'M') and (cd_education_status = 'Unknown ')) or ((cd_marital_status = 'W') and (cd_education_status = 'Advanced Degree '))) (type: boolean) + Statistics: Num rows: 4 Data size: 88 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((cd_marital_status = 'M') and (cd_education_status = 'Unknown ')) or ((cd_marital_status = 'W') and (cd_education_status = 'Advanced Degree '))) (type: boolean) + Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select '3 is expected:',count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +PREHOOK: type: QUERY +PREHOOK: Input: default@customer_demographics +#### A masked pattern was here #### +POSTHOOK: query: select '3 is expected:',count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@customer_demographics +#### A masked pattern was here #### +3 is expected: 3 +PREHOOK: query: explain +select count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +PREHOOK: type: QUERY +PREHOOK: Input: default@customer_demographics +#### A masked pattern was here #### +POSTHOOK: query: explain +select count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@customer_demographics +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: customer_demographics + filterExpr: (struct(cd_marital_status,cd_education_status)) IN (const struct('M','Unknown '), const struct('W','Advanced Degree ')) (type: boolean) + Statistics: Num rows: 4 Data size: 88 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (struct(cd_marital_status,cd_education_status)) IN (const struct('M','Unknown '), const struct('W','Advanced Degree ')) (type: boolean) + Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select '3 is expected:',count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +PREHOOK: type: QUERY +PREHOOK: Input: default@customer_demographics +#### A masked pattern was here #### +POSTHOOK: query: select '3 is expected:',count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@customer_demographics +#### A masked pattern was here #### +3 is expected: 3 diff --git ql/src/test/results/clientpositive/in_typecheck_varchar.q.out ql/src/test/results/clientpositive/in_typecheck_varchar.q.out new file mode 100644 index 00000000000..2dcacef8653 --- /dev/null +++ ql/src/test/results/clientpositive/in_typecheck_varchar.q.out @@ -0,0 +1,185 @@ +PREHOOK: query: create table ax(s varchar(1),t varchar(10)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ax +POSTHOOK: query: create table ax(s varchar(1),t varchar(10)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ax +PREHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ax +POSTHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ax +POSTHOOK: Lineage: ax.s SCRIPT [] +POSTHOOK: Lineage: ax.t SCRIPT [] +PREHOOK: query: explain +select 'expected 1',count(*) from ax where s = 'a' and t = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: explain +select 'expected 1',count(*) from ax where s = 'a' and t = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: ax + filterExpr: ((CAST( s AS STRING) = 'a') and (CAST( t AS STRING) = 'a')) (type: boolean) + Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((CAST( s AS STRING) = 'a') and (CAST( t AS STRING) = 'a')) (type: boolean) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 'expected 1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select 'expected 1',count(*) from ax where s = 'a' and t = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 1',count(*) from ax where s = 'a' and t = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 1 1 +PREHOOK: query: explain +select 'expected 2',count(*) from ax where (s,t) in (('a','a'),('b','bb')) +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: explain +select 'expected 2',count(*) from ax where (s,t) in (('a','a'),('b','bb')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: ax + filterExpr: (struct(CAST( s AS STRING),CAST( t AS STRING))) IN (const struct('a','a'), const struct('b','bb')) (type: boolean) + Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (struct(CAST( s AS STRING),CAST( t AS STRING))) IN (const struct('a','a'), const struct('b','bb')) (type: boolean) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 'expected 2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select 'expected 2',count(*) from ax where (s,t) in (('a','a'),('b','bb')) +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 2',count(*) from ax where (s,t) in (('a','a'),('b','bb')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 2 2 +PREHOOK: query: select 'expected 0',count(*) from ax where t = 'a ' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 0',count(*) from ax where t = 'a ' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 0 0 +PREHOOK: query: select 'expected 0',count(*) from ax where t = 'a ' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 0',count(*) from ax where t = 'a ' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 0 0 +PREHOOK: query: select 'expected 0',count(*) from ax where t = 'a d' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 0',count(*) from ax where t = 'a d' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 0 0 diff --git ql/src/test/results/clientpositive/infer_const_type.q.out ql/src/test/results/clientpositive/infer_const_type.q.out index e1d7de54225..db3b0129516 100644 --- ql/src/test/results/clientpositive/infer_const_type.q.out +++ ql/src/test/results/clientpositive/infer_const_type.q.out @@ -61,10 +61,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: infertypes - filterExpr: ((ti = 127Y) and (si = 32767S) and (i = 12345) and (bi = -12345L) and (fl = 906) and (db = -307.0D) and (UDFToDouble(str) = 1234.0D)) (type: boolean) + filterExpr: ((ti = 127Y) and (si = 32767S) and (i = 12345) and (bi = -12345L) and (fl = 906.0) and (db = -307.0D) and (UDFToDouble(str) = 1234.0D)) (type: boolean) Statistics: Num rows: 1 Data size: 1170 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((UDFToDouble(str) = 1234.0D) and (bi = -12345L) and (db = -307.0D) and (fl = 906) and (i = 12345) and (si = 32767S) and (ti = 127Y)) (type: boolean) + predicate: ((UDFToDouble(str) = 1234.0D) and (bi = -12345L) and (db = -307.0D) and (fl = 906.0) and (i = 12345) and (si = 32767S) and (ti = 127Y)) (type: boolean) Statistics: Num rows: 1 Data size: 1170 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 127Y (type: tinyint), 32767S (type: smallint), 12345 (type: int), -12345L (type: bigint), 906.0 (type: float), -307.0D (type: double), str (type: string) @@ -139,10 +139,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: infertypes - filterExpr: ((UDFToDouble(ti) = 128.0D) or (UDFToInteger(si) = 32768) or (UDFToDouble(i) = 2.147483648E9D) or (UDFToDouble(bi) = 9.223372036854776E18D)) (type: boolean) + filterExpr: ((UDFToDouble(ti) = 128.0D) or (UDFToInteger(si) = 32768) or (UDFToDouble(i) = 2.147483648E9D) or (UDFToDouble(bi) = 9.223372036854776E18D) or null) (type: boolean) Statistics: Num rows: 1 Data size: 1170 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((UDFToDouble(bi) = 9.223372036854776E18D) or (UDFToDouble(i) = 2.147483648E9D) or (UDFToDouble(ti) = 128.0D) or (UDFToInteger(si) = 32768)) (type: boolean) + predicate: ((UDFToDouble(bi) = 9.223372036854776E18D) or (UDFToDouble(i) = 2.147483648E9D) or (UDFToDouble(ti) = 128.0D) or (UDFToInteger(si) = 32768) or null) (type: boolean) Statistics: Num rows: 1 Data size: 1170 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ti (type: tinyint), si (type: smallint), i (type: int), bi (type: bigint), fl (type: float), db (type: double), str (type: string) diff --git ql/src/test/results/clientpositive/join45.q.out ql/src/test/results/clientpositive/join45.q.out index 47aaf7d0abc..6cf6c330f1c 100644 --- ql/src/test/results/clientpositive/join45.q.out +++ ql/src/test/results/clientpositive/join45.q.out @@ -714,8 +714,8 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 12500 Data size: 240800 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (struct(_col0,_col2)) IN (const struct(100,100), const struct(101,101), const struct(102,102)) (type: boolean) - Statistics: Num rows: 9375 Data size: 180600 Basic stats: COMPLETE Column stats: NONE + predicate: (struct(UDFToDouble(_col0),UDFToDouble(_col2))) IN (const struct(100.0D,100.0D), const struct(101.0D,101.0D), const struct(102.0D,102.0D)) (type: boolean) + Statistics: Num rows: 6250 Data size: 120400 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 190 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/join45X.q.out ql/src/test/results/clientpositive/join45X.q.out new file mode 100644 index 00000000000..880a2e3a3f7 --- /dev/null +++ ql/src/test/results/clientpositive/join45X.q.out @@ -0,0 +1,256 @@ +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +PREHOOK: query: EXPLAIN +SELECT * +FROM src1 JOIN src +ON (src1.key= 100 and src.key=100) +LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * +FROM src1 JOIN src +ON (src1.key= 100 and src.key=100) +LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src1 + filterExpr: (UDFToDouble(key) = 100.0D) (type: boolean) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) = 100.0D) (type: boolean) + Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + TableScan + alias: src + filterExpr: (UDFToDouble(key) = 100.0D) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) = 100.0D) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 3000 Data size: 57622 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 190 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 190 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + ListSink + +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +PREHOOK: query: EXPLAIN +SELECT * +FROM src1 JOIN src +ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) +LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * +FROM src1 JOIN src +ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) +LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 12500 Data size: 240800 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToDouble(_col0) = 100.0D) and (UDFToDouble(_col2) = 100.0D)) or ((UDFToDouble(_col0) = 101.0D) and (UDFToDouble(_col2) = 101.0D)) or ((UDFToDouble(_col0) = 102.0D) and (UDFToDouble(_col2) = 102.0D))) (type: boolean) + Statistics: Num rows: 9375 Data size: 180600 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 190 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 190 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + ListSink + +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +PREHOOK: query: SELECT * +FROM src1 JOIN src +ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) +LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM src1 JOIN src +ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) +LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +PREHOOK: query: EXPLAIN + SELECT * + FROM src1 JOIN src + ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) + LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN + SELECT * + FROM src1 JOIN src + ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) + LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 12500 Data size: 240800 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToDouble(_col0) = 100.0D) and (UDFToDouble(_col2) = 100.0D)) or ((UDFToDouble(_col0) = 101.0D) and (UDFToDouble(_col2) = 101.0D)) or ((UDFToDouble(_col0) = 102.0D) and (UDFToDouble(_col2) = 102.0D))) (type: boolean) + Statistics: Num rows: 9375 Data size: 180600 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 190 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 190 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + ListSink + diff --git ql/src/test/results/clientpositive/join47.q.out ql/src/test/results/clientpositive/join47.q.out index 4d9e937815e..2892b8b0566 100644 --- ql/src/test/results/clientpositive/join47.q.out +++ ql/src/test/results/clientpositive/join47.q.out @@ -706,8 +706,8 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3 - residual filter predicates: {(struct(_col0,_col2)) IN (const struct(100,100), const struct(101,101), const struct(102,102))} - Statistics: Num rows: 9375 Data size: 180600 Basic stats: COMPLETE Column stats: NONE + residual filter predicates: {(struct(UDFToDouble(_col0),UDFToDouble(_col2))) IN (const struct(100.0D,100.0D), const struct(101.0D,101.0D), const struct(102.0D,102.0D))} + Statistics: Num rows: 6250 Data size: 120400 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 190 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out index 5bd3d90249b..869aa18cbc4 100644 --- ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out +++ ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out @@ -137,20 +137,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart_acid - filterExpr: ((UDFToInteger(key)) IN (413, 43) and (hr = '11')) (type: boolean) + filterExpr: (((UDFToInteger(key) = 413) or (UDFToInteger(key) = 43)) and (hr = '11')) (type: boolean) Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator - predicate: (UDFToInteger(key)) IN (413, 43) (type: boolean) - Statistics: Num rows: 500 Data size: 181000 Basic stats: COMPLETE Column stats: PARTIAL + predicate: ((UDFToInteger(key) = 413) or (UDFToInteger(key) = 43)) (type: boolean) + Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ROW__ID (type: struct), key (type: string), concat(value, 'updated') (type: string), ds (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1000 Data size: 617000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: struct) sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1000 Data size: 617000 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) Execution mode: llap LLAP IO: may be used (ACID table) @@ -160,10 +160,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1000 Data size: 617000 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1000 Data size: 617000 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -798,20 +798,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart_acidb - filterExpr: ((UDFToInteger(key)) IN (413, 43) and (hr = '11')) (type: boolean) + filterExpr: (((UDFToInteger(key) = 413) or (UDFToInteger(key) = 43)) and (hr = '11')) (type: boolean) Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator - predicate: (UDFToInteger(key)) IN (413, 43) (type: boolean) - Statistics: Num rows: 500 Data size: 181000 Basic stats: COMPLETE Column stats: PARTIAL + predicate: ((UDFToInteger(key) = 413) or (UDFToInteger(key) = 43)) (type: boolean) + Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ROW__ID (type: struct), key (type: string), concat(value, 'updated') (type: string), ds (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1000 Data size: 617000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: struct) sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1000 Data size: 617000 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) Execution mode: llap LLAP IO: may be used (ACID table) @@ -821,10 +821,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1000 Data size: 617000 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1000 Data size: 617000 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1343,7 +1343,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterLongColumnInList(col 5:int, values [413, 43])(children: CastStringToLong(col 0:string) -> 5:int) + predicateExpression: FilterExprOrExpr(children: FilterLongColEqualLongScalar(col 5:int, val 413)(children: CastStringToLong(col 0:string) -> 5:int), FilterLongColEqualLongScalar(col 5:int, val 43)(children: CastStringToLong(col 0:string) -> 5:int)) Select Vectorization: className: VectorSelectOperator native: true @@ -2193,7 +2193,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterLongColumnInList(col 5:int, values [413, 43])(children: CastStringToLong(col 0:string) -> 5:int) + predicateExpression: FilterExprOrExpr(children: FilterLongColEqualLongScalar(col 5:int, val 413)(children: CastStringToLong(col 0:string) -> 5:int), FilterLongColEqualLongScalar(col 5:int, val 43)(children: CastStringToLong(col 0:string) -> 5:int)) Select Vectorization: className: VectorSelectOperator native: true diff --git ql/src/test/results/clientpositive/llap/bucketpruning1.q.out ql/src/test/results/clientpositive/llap/bucketpruning1.q.out index 9422eb49781..934b383af54 100644 --- ql/src/test/results/clientpositive/llap/bucketpruning1.q.out +++ ql/src/test/results/clientpositive/llap/bucketpruning1.q.out @@ -1509,7 +1509,7 @@ POSTHOOK: Input: default@srcbucket_pruned #### A masked pattern was here #### OPTIMIZED SQL: SELECT `key`, `value`, `ds` FROM `default`.`srcbucket_pruned` -WHERE FALSE +WHERE NULL STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 diff --git ql/src/test/results/clientpositive/llap/check_constraint.q.out ql/src/test/results/clientpositive/llap/check_constraint.q.out index 9b1f8def6eb..72736807cc2 100644 --- ql/src/test/results/clientpositive/llap/check_constraint.q.out +++ ql/src/test/results/clientpositive/llap/check_constraint.q.out @@ -116,7 +116,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: enforce_constraint((((((((- _col0) > (- 10)) is not false and (_col1 > 10) is not false) and _col2 is not null is not false) and _col3 BETWEEN _col0 AND _col1 is not false) and (_col4) IN (23.4, 56, 4) is not false) and ((_col5 > round(567.6)) and (_col5 < round(1000.4))) is not false)) (type: boolean) + predicate: enforce_constraint((((((((- _col0) > (- 10)) is not false and (_col1 > 10) is not false) and _col2 is not null is not false) and _col3 BETWEEN _col0 AND _col1 is not false) and ((_col4 = 23.4) or (_col4 = 56) or (_col4 = 4)) is not false) and ((_col5 > round(567.6)) and (_col5 < round(1000.4))) is not false)) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: @@ -3194,7 +3194,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: enforce_constraint((null) IN (4, 5) is not false) (type: boolean) + predicate: enforce_constraint(((null = 4) or (null = 5)) is not false) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: @@ -3273,7 +3273,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: enforce_constraint((_col1) IN (4, 5) is not false) (type: boolean) + predicate: enforce_constraint(((_col1 = 4) or (_col1 = 5)) is not false) (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: diff --git ql/src/test/results/clientpositive/llap/dec_str.q.out ql/src/test/results/clientpositive/llap/dec_str.q.out index 554031e952e..ce509a7597d 100644 --- ql/src/test/results/clientpositive/llap/dec_str.q.out +++ ql/src/test/results/clientpositive/llap/dec_str.q.out @@ -102,11 +102,10 @@ STAGE PLANS: Processor Tree: TableScan alias: t1 - filterExpr: (a = null) (type: boolean) Filter Operator - predicate: (a = null) (type: boolean) + predicate: false (type: boolean) Select Operator - expressions: null (type: decimal(3,1)) + expressions: a (type: decimal(3,1)) outputColumnNames: _col0 ListSink @@ -128,11 +127,10 @@ STAGE PLANS: Processor Tree: TableScan alias: t1 - filterExpr: (a = null) (type: boolean) Filter Operator - predicate: (a = null) (type: boolean) + predicate: false (type: boolean) Select Operator - expressions: null (type: decimal(3,1)) + expressions: a (type: decimal(3,1)) outputColumnNames: _col0 ListSink diff --git ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out index 40dc5e97e5a..a52d939de9f 100644 --- ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out +++ ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out @@ -202,33 +202,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_part - filterExpr: ((key = 'foo') and (ds) IN ('2008-04-08')) (type: boolean) - Statistics: Num rows: 1601 Data size: 444998 Basic stats: COMPLETE Column stats: PARTIAL + filterExpr: ((key = 'foo') and (ds = '2008-04-08')) (type: boolean) + Statistics: Num rows: 1601 Data size: 150414 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (key = 'foo') (type: boolean) - Statistics: Num rows: 5 Data size: 1355 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: ROW__ID (type: struct), ds (type: string) - outputColumnNames: _col0, _col3 - Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL + expressions: ROW__ID (type: struct) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: struct) sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col3 (type: string) + Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: may be used (ACID table) Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: struct), 'foo' (type: string), 'bar' (type: string), VALUE._col1 (type: string) + expressions: KEY.reducesinkkey0 (type: struct), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -502,32 +501,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_part_sdpo - filterExpr: ((key = 'foo') and (ds) IN ('2008-04-08')) (type: boolean) - Statistics: Num rows: 1601 Data size: 444998 Basic stats: COMPLETE Column stats: PARTIAL + filterExpr: ((key = 'foo') and (ds = '2008-04-08')) (type: boolean) + Statistics: Num rows: 1601 Data size: 150414 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (key = 'foo') (type: boolean) - Statistics: Num rows: 5 Data size: 1355 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: ROW__ID (type: struct), ds (type: string) - outputColumnNames: _col0, _col3 - Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL + expressions: ROW__ID (type: struct) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: string), _bucket_number (type: string), _col0 (type: struct) - sort order: +++ - Map-reduce partition columns: _col3 (type: string) - value expressions: 'foo' (type: string), 'bar' (type: string) + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: may be used (ACID table) Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY._col0 (type: struct), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._bucket_number (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number + expressions: KEY.reducesinkkey0 (type: struct), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git ql/src/test/results/clientpositive/llap/explainuser_1.q.out ql/src/test/results/clientpositive/llap/explainuser_1.q.out index f240468558e..71838eeda55 100644 --- ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -484,7 +484,7 @@ Stage-0 Group By Operator [GBY_6] (rows=2 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_37] (rows=5 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 8 [SIMPLE_EDGE] llap @@ -500,7 +500,7 @@ Stage-0 Group By Operator [GBY_13] (rows=2 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_38] (rows=5 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -574,7 +574,7 @@ Stage-0 Group By Operator [GBY_6] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_37] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 8 [SIMPLE_EDGE] llap @@ -590,7 +590,7 @@ Stage-0 Group By Operator [GBY_13] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_38] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -657,7 +657,7 @@ Stage-0 Group By Operator [GBY_6] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_34] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 7 [SIMPLE_EDGE] llap @@ -673,7 +673,7 @@ Stage-0 Group By Operator [GBY_13] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_35] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -745,7 +745,7 @@ Stage-0 Group By Operator [GBY_6] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_36] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 8 [SIMPLE_EDGE] llap @@ -761,7 +761,7 @@ Stage-0 Group By Operator [GBY_13] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_37] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -828,7 +828,7 @@ Stage-0 Group By Operator [GBY_6] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_34] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 7 [SIMPLE_EDGE] llap @@ -844,7 +844,7 @@ Stage-0 Group By Operator [GBY_13] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_35] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -1315,7 +1315,7 @@ Stage-0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1","_col2"] Filter Operator [FIL_21] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 3 [SIMPLE_EDGE] llap @@ -1324,7 +1324,7 @@ Stage-0 Select Operator [SEL_5] (rows=9 width=89) Output:["_col0","_col1"] Filter Operator [FIL_22] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 4 [SIMPLE_EDGE] llap @@ -1374,7 +1374,7 @@ Stage-0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1","_col2"] Filter Operator [FIL_21] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 3 [SIMPLE_EDGE] llap @@ -1383,7 +1383,7 @@ Stage-0 Select Operator [SEL_5] (rows=9 width=89) Output:["_col0","_col1"] Filter Operator [FIL_22] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 4 [SIMPLE_EDGE] llap @@ -1640,7 +1640,7 @@ Stage-0 Group By Operator [GBY_3] (rows=3 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_50] (rows=6 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0))) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) TableScan [TS_0] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 9 [SIMPLE_EDGE] llap @@ -1664,7 +1664,7 @@ Stage-0 Group By Operator [GBY_16] (rows=3 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_52] (rows=6 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0))) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) TableScan [TS_13] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -1703,7 +1703,7 @@ Stage-0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1"] Filter Operator [FIL_15] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 3 [SIMPLE_EDGE] llap @@ -1755,7 +1755,7 @@ Stage-0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1","_col2"] Filter Operator [FIL_25] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 3 [SIMPLE_EDGE] llap @@ -1766,7 +1766,7 @@ Stage-0 Select Operator [SEL_5] (rows=9 width=85) Output:["_col0"] Filter Operator [FIL_26] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 4 [SIMPLE_EDGE] llap @@ -1855,7 +1855,7 @@ Stage-0 Group By Operator [GBY_3] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_41] (rows=1 width=93) - predicate:((((c_int + 1) + 1) >= 0) and (((c_int + 1) > 0) or (UDFToDouble(key) >= 0.0D)) and ((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (UDFToDouble(key) > 0.0D) and (c_float > 0)) + predicate:((((c_int + 1) + 1) >= 0) and (((c_int + 1) > 0) or (UDFToDouble(key) >= 0.0D)) and ((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (UDFToDouble(key) > 0.0D) and (c_float > 0.0)) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 7 [SIMPLE_EDGE] llap @@ -1873,7 +1873,7 @@ Stage-0 Group By Operator [GBY_12] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_42] (rows=1 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (UDFToDouble(key) > 0.0D) and (c_float > 0)) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (UDFToDouble(key) > 0.0D) and (c_float > 0.0)) TableScan [TS_9] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] diff --git ql/src/test/results/clientpositive/llap/is_distinct_from.q.out ql/src/test/results/clientpositive/llap/is_distinct_from.q.out index f5a34750ce3..010c4a15f85 100644 --- ql/src/test/results/clientpositive/llap/is_distinct_from.q.out +++ ql/src/test/results/clientpositive/llap/is_distinct_from.q.out @@ -247,7 +247,7 @@ STAGE PLANS: Filter Operator predicate: (y = null) (type: boolean) Select Operator - expressions: x (type: string), y (type: string) + expressions: x (type: string), null (type: string) outputColumnNames: _col0, _col1 ListSink diff --git ql/src/test/results/clientpositive/llap/lineage3.q.out ql/src/test/results/clientpositive/llap/lineage3.q.out index 9bec309c9c9..94c6a13c99c 100644 --- ql/src/test/results/clientpositive/llap/lineage3.q.out +++ ql/src/test/results/clientpositive/llap/lineage3.q.out @@ -317,7 +317,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc PREHOOK: Input: default@dest_v3 #### A masked pattern was here #### -{"version":"1.0","engine":"tez","database":"default","hash":"fd4e0dd59f42b53fc07125817451df49","queryText":"select * from dest_v3 limit 2","edges":[{"sources":[3,4,5,6,7],"targets":[0],"expression":"(tok_function sum (. (tok_table_or_col $hdt$_0) ctinyint) (tok_windowspec (tok_partitioningspec (tok_distributeby (. (tok_table_or_col $hdt$_0) csmallint)) (tok_orderby (tok_tabsortcolnameasc (tok_nulls_last (. (tok_table_or_col $hdt$_0) csmallint))))) (tok_windowvalues (preceding 2147483647) current)))","edgeType":"PROJECTION"},{"sources":[6],"targets":[1],"expression":"count(default.alltypesorc.cstring1)","edgeType":"PROJECTION"},{"sources":[5],"targets":[2],"edgeType":"PROJECTION"},{"sources":[8,7],"targets":[0,1,2],"expression":"(a.cboolean2 and a.cint is not null)","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2],"expression":"(a.cint = b.cint)","edgeType":"PREDICATE"},{"sources":[9,7],"targets":[0,1,2],"expression":"((b.cfloat > 0) and b.cint is not null)","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2],"expression":"(count(default.alltypesorc.cint) > 10L)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"dest_v3.a"},{"id":1,"vertexType":"COLUMN","vertexId":"dest_v3.x"},{"id":2,"vertexType":"COLUMN","vertexId":"dest_v3.cboolean1"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.csmallint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":6,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":7,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":8,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean2"},{"id":9,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cfloat"}]} +{"version":"1.0","engine":"tez","database":"default","hash":"fd4e0dd59f42b53fc07125817451df49","queryText":"select * from dest_v3 limit 2","edges":[{"sources":[3,4,5,6,7],"targets":[0],"expression":"(tok_function sum (. (tok_table_or_col $hdt$_0) ctinyint) (tok_windowspec (tok_partitioningspec (tok_distributeby (. (tok_table_or_col $hdt$_0) csmallint)) (tok_orderby (tok_tabsortcolnameasc (tok_nulls_last (. (tok_table_or_col $hdt$_0) csmallint))))) (tok_windowvalues (preceding 2147483647) current)))","edgeType":"PROJECTION"},{"sources":[6],"targets":[1],"expression":"count(default.alltypesorc.cstring1)","edgeType":"PROJECTION"},{"sources":[5],"targets":[2],"edgeType":"PROJECTION"},{"sources":[8,7],"targets":[0,1,2],"expression":"(a.cboolean2 and a.cint is not null)","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2],"expression":"(a.cint = b.cint)","edgeType":"PREDICATE"},{"sources":[9,7],"targets":[0,1,2],"expression":"((b.cfloat > 0.0) and b.cint is not null)","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2],"expression":"(count(default.alltypesorc.cint) > 10L)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"dest_v3.a"},{"id":1,"vertexType":"COLUMN","vertexId":"dest_v3.x"},{"id":2,"vertexType":"COLUMN","vertexId":"dest_v3.cboolean1"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.csmallint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":6,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":7,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":8,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean2"},{"id":9,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cfloat"}]} 38 216 false 38 229 true PREHOOK: query: drop table if exists src_dp diff --git ql/src/test/results/clientpositive/llap/multi_column_in.q.out ql/src/test/results/clientpositive/llap/multi_column_in.q.out index 414be226e6a..015d78764a2 100644 --- ql/src/test/results/clientpositive/llap/multi_column_in.q.out +++ ql/src/test/results/clientpositive/llap/multi_column_in.q.out @@ -121,11 +121,11 @@ STAGE PLANS: Processor Tree: TableScan alias: emps_n1 - filterExpr: (struct((empno + 1),deptno)) IN (const struct(1,2), const struct(3,2)) (type: boolean) + filterExpr: (((empno + 1)) IN (1, 3) and (deptno = 2)) (type: boolean) Filter Operator - predicate: (struct((empno + 1),deptno)) IN (const struct(1,2), const struct(3,2)) (type: boolean) + predicate: (((empno + 1)) IN (1, 3) and (deptno = 2)) (type: boolean) Select Operator - expressions: empno (type: int), deptno (type: int), empname (type: string) + expressions: empno (type: int), 2 (type: int), empname (type: string) outputColumnNames: _col0, _col1, _col2 ListSink @@ -149,9 +149,9 @@ STAGE PLANS: Processor Tree: TableScan alias: emps_n1 - filterExpr: (not (struct((empno + 1),deptno)) IN (const struct(1,2), const struct(3,2))) (type: boolean) + filterExpr: ((((empno + 1) <> 1) or (deptno <> 2)) and (((empno + 1) <> 3) or (deptno <> 2))) (type: boolean) Filter Operator - predicate: (not (struct((empno + 1),deptno)) IN (const struct(1,2), const struct(3,2))) (type: boolean) + predicate: ((((empno + 1) <> 1) or (deptno <> 2)) and (((empno + 1) <> 3) or (deptno <> 2))) (type: boolean) Select Operator expressions: empno (type: int), deptno (type: int), empname (type: string) outputColumnNames: _col0, _col1, _col2 @@ -221,11 +221,11 @@ STAGE PLANS: Processor Tree: TableScan alias: emps_n1 - filterExpr: (struct(((empno * 2) | 1),deptno)) IN (struct((empno + 1),2), struct((empno + 2),2)) (type: boolean) + filterExpr: ((deptno = 2) and ((((empno * 2) | 1) = (empno + 1)) or (((empno * 2) | 1) = (empno + 2)))) (type: boolean) Filter Operator - predicate: (struct(((empno * 2) | 1),deptno)) IN (struct((empno + 1),2), struct((empno + 2),2)) (type: boolean) + predicate: (((((empno * 2) | 1) = (empno + 1)) or (((empno * 2) | 1) = (empno + 2))) and (deptno = 2)) (type: boolean) Select Operator - expressions: empno (type: int), deptno (type: int), empname (type: string) + expressions: empno (type: int), 2 (type: int), empname (type: string) outputColumnNames: _col0, _col1, _col2 ListSink diff --git ql/src/test/results/clientpositive/llap/multi_column_in_single.q.out ql/src/test/results/clientpositive/llap/multi_column_in_single.q.out index db9c676a814..ec3232e8291 100644 --- ql/src/test/results/clientpositive/llap/multi_column_in_single.q.out +++ ql/src/test/results/clientpositive/llap/multi_column_in_single.q.out @@ -131,11 +131,11 @@ STAGE PLANS: Processor Tree: TableScan alias: emps_n7 - filterExpr: (struct((empno + 1),deptno)) IN (const struct(3,2)) (type: boolean) + filterExpr: ((deptno = 2) and ((empno + 1) = 3)) (type: boolean) Filter Operator - predicate: (struct((empno + 1),deptno)) IN (const struct(3,2)) (type: boolean) + predicate: (((empno + 1) = 3) and (deptno = 2)) (type: boolean) Select Operator - expressions: empno (type: int), deptno (type: int), empname (type: string) + expressions: empno (type: int), 2 (type: int), empname (type: string) outputColumnNames: _col0, _col1, _col2 ListSink @@ -159,9 +159,9 @@ STAGE PLANS: Processor Tree: TableScan alias: emps_n7 - filterExpr: (not (struct((empno + 1),deptno)) IN (const struct(3,2))) (type: boolean) + filterExpr: (((empno + 1) <> 3) or (deptno <> 2)) (type: boolean) Filter Operator - predicate: (not (struct((empno + 1),deptno)) IN (const struct(3,2))) (type: boolean) + predicate: (((empno + 1) <> 3) or (deptno <> 2)) (type: boolean) Select Operator expressions: empno (type: int), deptno (type: int), empname (type: string) outputColumnNames: _col0, _col1, _col2 @@ -185,11 +185,11 @@ STAGE PLANS: Processor Tree: TableScan alias: emps_n7 - filterExpr: (struct(((empno * 2) | 1),deptno)) IN (struct((empno + 2),2)) (type: boolean) + filterExpr: ((deptno = 2) and (((empno * 2) | 1) = (empno + 2))) (type: boolean) Filter Operator - predicate: (struct(((empno * 2) | 1),deptno)) IN (struct((empno + 2),2)) (type: boolean) + predicate: ((((empno * 2) | 1) = (empno + 2)) and (deptno = 2)) (type: boolean) Select Operator - expressions: empno (type: int), deptno (type: int), empname (type: string) + expressions: empno (type: int), 2 (type: int), empname (type: string) outputColumnNames: _col0, _col1, _col2 ListSink diff --git ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out index 3b163794b60..56274908191 100644 --- ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out +++ ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out @@ -902,12 +902,12 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 - RECORDS_OUT_OPERATOR_FIL_7: 0 - RECORDS_OUT_OPERATOR_FS_9: 1 - RECORDS_OUT_OPERATOR_GBY_8: 1 + RECORDS_OUT_OPERATOR_FIL_8: 0 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 RECORDS_OUT_OPERATOR_MAP_0: 0 - RECORDS_OUT_OPERATOR_RS_3: 0 - RECORDS_OUT_OPERATOR_SEL_2: 0 + RECORDS_OUT_OPERATOR_RS_10: 0 + RECORDS_OUT_OPERATOR_SEL_9: 0 RECORDS_OUT_OPERATOR_TS_0: 0 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 823 diff --git ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out index 21180a3da8c..797eaa8324e 100644 --- ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out +++ ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out @@ -619,16 +619,16 @@ STAGE PLANS: alias: orc_pred Statistics: Num rows: 1049 Data size: 105941 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((not (t) IN (-1, -2, -3)) and (s like 'bob%') and s is not null and t BETWEEN 25 AND 30) (type: boolean) - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((s like 'bob%') and (t <> -1) and (t <> -2) and (t <> -3) and s is not null and t BETWEEN 25 AND 30) (type: boolean) + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: t (type: tinyint), s (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: tinyint), _col1 (type: string) sort order: ++ - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs Reducer 2 @@ -637,10 +637,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -686,19 +686,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: orc_pred - filterExpr: (s is not null and (s like 'bob%') and (not (t) IN (-1, -2, -3)) and t BETWEEN 25 AND 30) (type: boolean) + filterExpr: (s is not null and (s like 'bob%') and (t <> -1) and (t <> -2) and (t <> -3) and t BETWEEN 25 AND 30) (type: boolean) Statistics: Num rows: 1049 Data size: 105941 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((not (t) IN (-1, -2, -3)) and (s like 'bob%') and s is not null and t BETWEEN 25 AND 30) (type: boolean) - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((s like 'bob%') and (t <> -1) and (t <> -2) and (t <> -3) and s is not null and t BETWEEN 25 AND 30) (type: boolean) + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: t (type: tinyint), s (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: tinyint), _col1 (type: string) sort order: ++ - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs Reducer 2 @@ -707,10 +707,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out index 8330e93115c..81af3d4ef24 100644 --- ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out +++ ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out @@ -557,16 +557,16 @@ STAGE PLANS: alias: tbl_pred Statistics: Num rows: 1049 Data size: 105941 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((not (t) IN (-1, -2, -3)) and (s like 'bob%') and s is not null and t BETWEEN 25 AND 30) (type: boolean) - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((s like 'bob%') and (t <> -1) and (t <> -2) and (t <> -3) and s is not null and t BETWEEN 25 AND 30) (type: boolean) + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: t (type: tinyint), s (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: tinyint), _col1 (type: string) sort order: ++ - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs (cache only) Reducer 2 @@ -575,10 +575,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -624,19 +624,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_pred - filterExpr: (s is not null and (s like 'bob%') and (not (t) IN (-1, -2, -3)) and t BETWEEN 25 AND 30) (type: boolean) + filterExpr: (s is not null and (s like 'bob%') and (t <> -1) and (t <> -2) and (t <> -3) and t BETWEEN 25 AND 30) (type: boolean) Statistics: Num rows: 1049 Data size: 105941 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((not (t) IN (-1, -2, -3)) and (s like 'bob%') and s is not null and t BETWEEN 25 AND 30) (type: boolean) - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((s like 'bob%') and (t <> -1) and (t <> -2) and (t <> -3) and s is not null and t BETWEEN 25 AND 30) (type: boolean) + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: t (type: tinyint), s (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: tinyint), _col1 (type: string) sort order: ++ - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs (cache only) Reducer 2 @@ -645,10 +645,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 24 Data size: 2424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2525 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git ql/src/test/results/clientpositive/llap/vector_between_in.q.out ql/src/test/results/clientpositive/llap/vector_between_in.q.out index e9ea4611acc..12711930550 100644 --- ql/src/test/results/clientpositive/llap/vector_between_in.q.out +++ ql/src/test/results/clientpositive/llap/vector_between_in.q.out @@ -1144,13 +1144,13 @@ STAGE PLANS: TableScan Vectorization: native: true Select Operator - expressions: ((cdate = DATE'1969-10-26') or (cdate = DATE'1969-07-14')) (type: boolean) + expressions: (cdate) IN (DATE'1969-10-26', DATE'1969-07-14') (type: boolean) outputColumnNames: _col0 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [7] - selectExpressions: ColOrCol(col 5:boolean, col 6:boolean)(children: DateColEqualDateScalar(col 3:date, date 1969-10-26) -> 5:boolean, DateColEqualDateScalar(col 3:date, date 1969-07-14) -> 6:boolean) -> 7:boolean + projectedOutputColumnNums: [5] + selectExpressions: LongColumnInList(col 3, values [-67, -171]) -> 5:boolean Statistics: Num rows: 12289 Data size: 653856 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1158,7 +1158,7 @@ STAGE PLANS: aggregators: VectorUDAFCountStar(*) -> bigint className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 7:boolean + keyExpressions: col 5:boolean native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -1286,13 +1286,13 @@ STAGE PLANS: TableScan Vectorization: native: true Select Operator - expressions: ((cdecimal1 = 2365.8945945946) or (cdecimal1 = 881.0135135135) or (cdecimal1 = -3367.6517567568)) (type: boolean) + expressions: (cdecimal1) IN (2365.8945945946, 881.0135135135, -3367.6517567568) (type: boolean) outputColumnNames: _col0 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [8] - selectExpressions: VectorUDFAdaptor(((cdecimal1 = 2365.8945945946) or (cdecimal1 = 881.0135135135) or (cdecimal1 = -3367.6517567568)))(children: DecimalColEqualDecimalScalar(col 1:decimal(20,10), val 2365.8945945946) -> 5:boolean, DecimalColEqualDecimalScalar(col 1:decimal(20,10), val 881.0135135135) -> 6:boolean, DecimalColEqualDecimalScalar(col 1:decimal(20,10), val -3367.6517567568) -> 7:boolean) -> 8:boolean + projectedOutputColumnNums: [5] + selectExpressions: DecimalColumnInList(col 1:decimal(20,10), values [2365.8945945946, 881.0135135135, -3367.6517567568]) -> 5:boolean Statistics: Num rows: 12289 Data size: 1307712 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1300,7 +1300,7 @@ STAGE PLANS: aggregators: VectorUDAFCountStar(*) -> bigint className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 8:boolean + keyExpressions: col 5:boolean native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -1327,7 +1327,7 @@ STAGE PLANS: featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reducer 2 Execution mode: vectorized, llap diff --git ql/src/test/results/clientpositive/llap/vector_struct_in.q.out ql/src/test/results/clientpositive/llap/vector_struct_in.q.out index 4dfa80e9843..185c312eb34 100644 --- ql/src/test/results/clientpositive/llap/vector_struct_in.q.out +++ ql/src/test/results/clientpositive/llap/vector_struct_in.q.out @@ -869,7 +869,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test_4 - filterExpr: (struct(my_bigint,my_string,my_double)) IN (const struct(1L,'a',1.5D), const struct(1L,'b',-0.5D), const struct(3L,'b',1.5D), const struct(1L,'d',1.5D), const struct(1L,'c',1.5D), const struct(1L,'b',2.5D), const struct(1L,'b',0.5D), const struct(5L,'b',1.5D), const struct(1L,'a',0.5D), const struct(3L,'b',1.5D)) (type: boolean) + filterExpr: (struct(my_bigint,my_string,my_double)) IN (const struct(1L,'a',1.5D), const struct(1L,'b',-0.5D), const struct(3L,'b',1.5D), const struct(1L,'d',1.5D), const struct(1L,'c',1.5D), const struct(1L,'b',2.5D), const struct(1L,'b',0.5D), const struct(5L,'b',1.5D), const struct(1L,'a',0.5D)) (type: boolean) Statistics: Num rows: 3 Data size: 303 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -878,7 +878,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterStructColumnInList(structExpressions [col 0:bigint, col 1:string, col 2:double], fieldVectorColumnTypes [LONG, BYTES, DOUBLE], structColumnMap [0, 1, 2]) - predicate: (struct(my_bigint,my_string,my_double)) IN (const struct(1L,'a',1.5D), const struct(1L,'b',-0.5D), const struct(3L,'b',1.5D), const struct(1L,'d',1.5D), const struct(1L,'c',1.5D), const struct(1L,'b',2.5D), const struct(1L,'b',0.5D), const struct(5L,'b',1.5D), const struct(1L,'a',0.5D), const struct(3L,'b',1.5D)) (type: boolean) + predicate: (struct(my_bigint,my_string,my_double)) IN (const struct(1L,'a',1.5D), const struct(1L,'b',-0.5D), const struct(3L,'b',1.5D), const struct(1L,'d',1.5D), const struct(1L,'c',1.5D), const struct(1L,'b',2.5D), const struct(1L,'b',0.5D), const struct(5L,'b',1.5D), const struct(1L,'a',0.5D)) (type: boolean) Statistics: Num rows: 3 Data size: 303 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: my_bigint (type: bigint), my_string (type: string), my_double (type: double) @@ -1004,7 +1004,7 @@ STAGE PLANS: TableScan Vectorization: native: true Select Operator - expressions: my_bigint (type: bigint), my_string (type: string), my_double (type: double), (struct(my_bigint,my_string,my_double)) IN (const struct(1L,'a',1.5D), const struct(1L,'b',-0.5D), const struct(3L,'b',1.5D), const struct(1L,'d',1.5D), const struct(1L,'c',1.5D), const struct(1L,'b',2.5D), const struct(1L,'b',0.5D), const struct(5L,'b',1.5D), const struct(1L,'a',0.5D), const struct(3L,'b',1.5D)) (type: boolean) + expressions: my_bigint (type: bigint), my_string (type: string), my_double (type: double), (struct(my_bigint,my_string,my_double)) IN (const struct(1L,'a',1.5D), const struct(1L,'b',-0.5D), const struct(3L,'b',1.5D), const struct(1L,'d',1.5D), const struct(1L,'c',1.5D), const struct(1L,'b',2.5D), const struct(1L,'b',0.5D), const struct(5L,'b',1.5D), const struct(1L,'a',0.5D)) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3 Select Vectorization: className: VectorSelectOperator diff --git ql/src/test/results/clientpositive/llap/vectorization_13.q.out ql/src/test/results/clientpositive/llap/vectorization_13.q.out index 4ce654f960b..d84d0b42e34 100644 --- ql/src/test/results/clientpositive/llap/vectorization_13.q.out +++ ql/src/test/results/clientpositive/llap/vectorization_13.q.out @@ -89,7 +89,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 2028982 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -99,7 +99,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28789.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28788.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 5461 Data size: 901772 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) @@ -448,7 +448,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 2028982 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -457,7 +457,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28801.388)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28801.336)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 5461 Data size: 901772 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) diff --git ql/src/test/results/clientpositive/llap/vectorization_6.q.out ql/src/test/results/clientpositive/llap/vectorization_6.q.out index a2f730beca2..2990bd4b827 100644 --- ql/src/test/results/clientpositive/llap/vectorization_6.q.out +++ ql/src/test/results/clientpositive/llap/vectorization_6.q.out @@ -61,7 +61,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257))))) (type: boolean) + filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0))))) (type: boolean) Statistics: Num rows: 12288 Data size: 2110130 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -71,7 +71,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprAndExpr(children: FilterLongColNotEqualLongScalar(col 0:tinyint, val 0), FilterExprOrExpr(children: FilterExprAndExpr(children: FilterLongColLessEqualLongScalar(col 10:boolean, val 0), FilterLongColGreaterEqualLongColumn(col 11:boolean, col 10:boolean)), FilterExprAndExpr(children: SelectColumnIsNotNull(col 3:bigint), FilterExprOrExpr(children: FilterStringColLikeStringScalar(col 7:string, pattern %a), FilterDoubleColLessEqualDoubleScalar(col 4:float, val -257.0))))) - predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257)))) and (ctinyint <> 0Y)) (type: boolean) + predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0)))) and (ctinyint <> 0Y)) (type: boolean) Statistics: Num rows: 5951 Data size: 1022000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: cboolean1 (type: boolean), cfloat (type: float), cstring1 (type: string), (988888 * UDFToInteger(csmallint)) (type: int), (- csmallint) (type: smallint), (- cfloat) (type: float), (-26.28D / UDFToDouble(cfloat)) (type: double), (cfloat * 359.0) (type: float), (cint % UDFToInteger(ctinyint)) (type: int), (- cdouble) (type: double), (UDFToInteger(ctinyint) - -75) (type: int), (762 * (cint % UDFToInteger(ctinyint))) (type: int) diff --git ql/src/test/results/clientpositive/llap/vectorization_8.q.out ql/src/test/results/clientpositive/llap/vectorization_8.q.out index 21ce7b8ebd0..a0e1a4b2145 100644 --- ql/src/test/results/clientpositive/llap/vectorization_8.q.out +++ ql/src/test/results/clientpositive/llap/vectorization_8.q.out @@ -72,7 +72,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D)) or (cfloat < -6432) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) + filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D)) or (cfloat < -6432.0) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 2983078 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -82,7 +82,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 10.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 16.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0))) - predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D))) (type: boolean) + predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432.0) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D))) (type: boolean) Statistics: Num rows: 3059 Data size: 742850 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) @@ -313,7 +313,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D)) or (cfloat < -6432) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) + filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D)) or (cfloat < -6432.0) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 2983078 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -322,7 +322,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 12.503)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 11.998)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0))) - predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D))) (type: boolean) + predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432.0) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D))) (type: boolean) Statistics: Num rows: 3059 Data size: 742850 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) diff --git ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out index 7f1c6a295e1..f9297067570 100644 --- ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out +++ ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out @@ -622,7 +622,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((ctimestamp1 = ctimestamp2) or (cfloat = 762) or (cstring1 = 'ss') or ((UDFToLong(csmallint) <= cbigint) and (cboolean2 = 1)) or (cboolean1 is not null and ctimestamp2 is not null and (cstring2 > 'a'))) (type: boolean) + filterExpr: ((ctimestamp1 = ctimestamp2) or (cfloat = 762.0) or (cstring1 = 'ss') or ((UDFToLong(csmallint) <= cbigint) and (cboolean2 = 1)) or (cboolean1 is not null and ctimestamp2 is not null and (cstring2 > 'a'))) (type: boolean) Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -631,7 +631,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterTimestampColEqualTimestampColumn(col 8:timestamp, col 9:timestamp), FilterDoubleColEqualDoubleScalar(col 4:float, val 762.0), FilterStringGroupColEqualStringScalar(col 6:string, val ss), FilterExprAndExpr(children: FilterLongColLessEqualLongColumn(col 1:bigint, col 3:bigint)(children: col 1:smallint), FilterLongColEqualLongScalar(col 11:boolean, val 1)), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), SelectColumnIsNotNull(col 9:timestamp), FilterStringGroupColGreaterStringScalar(col 7:string, val a))) - predicate: (((UDFToLong(csmallint) <= cbigint) and (cboolean2 = 1)) or (cboolean1 is not null and ctimestamp2 is not null and (cstring2 > 'a')) or (cfloat = 762) or (cstring1 = 'ss') or (ctimestamp1 = ctimestamp2)) (type: boolean) + predicate: (((UDFToLong(csmallint) <= cbigint) and (cboolean2 = 1)) or (cboolean1 is not null and ctimestamp2 is not null and (cstring2 > 'a')) or (cfloat = 762.0) or (cstring1 = 'ss') or (ctimestamp1 = ctimestamp2)) (type: boolean) Statistics: Num rows: 11346 Data size: 2856120 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: cbigint (type: bigint), ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cdouble (type: double), UDFToDouble(cbigint) (type: double), (UDFToDouble(cbigint) * UDFToDouble(cbigint)) (type: double), UDFToDouble(csmallint) (type: double), (UDFToDouble(csmallint) * UDFToDouble(csmallint)) (type: double), (cdouble * cdouble) (type: double) @@ -860,7 +860,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: (((ctimestamp2 <= ctimestamp1) and (UDFToDouble(cbigint) <> cdouble) and (cstring1 >= 'ss')) or ((csmallint < UDFToShort(ctinyint)) and (UDFToDouble(ctimestamp1) >= 0.0D)) or (cfloat = 17)) (type: boolean) + filterExpr: (((ctimestamp2 <= ctimestamp1) and (UDFToDouble(cbigint) <> cdouble) and (cstring1 >= 'ss')) or ((csmallint < UDFToShort(ctinyint)) and (UDFToDouble(ctimestamp1) >= 0.0D)) or (cfloat = 17.0)) (type: boolean) Statistics: Num rows: 12288 Data size: 2139070 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -869,7 +869,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterTimestampColLessEqualTimestampColumn(col 9:timestamp, col 8:timestamp), FilterDoubleColNotEqualDoubleColumn(col 13:double, col 5:double)(children: CastLongToDouble(col 3:bigint) -> 13:double), FilterStringGroupColGreaterEqualStringScalar(col 6:string, val ss)), FilterExprAndExpr(children: FilterLongColLessLongColumn(col 1:smallint, col 0:smallint)(children: col 0:tinyint), FilterDoubleColGreaterEqualDoubleScalar(col 13:double, val 0.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double)), FilterDoubleColEqualDoubleScalar(col 4:float, val 17.0)) - predicate: (((csmallint < UDFToShort(ctinyint)) and (UDFToDouble(ctimestamp1) >= 0.0D)) or ((ctimestamp2 <= ctimestamp1) and (UDFToDouble(cbigint) <> cdouble) and (cstring1 >= 'ss')) or (cfloat = 17)) (type: boolean) + predicate: (((csmallint < UDFToShort(ctinyint)) and (UDFToDouble(ctimestamp1) >= 0.0D)) or ((ctimestamp2 <= ctimestamp1) and (UDFToDouble(cbigint) <> cdouble) and (cstring1 >= 'ss')) or (cfloat = 17.0)) (type: boolean) Statistics: Num rows: 2824 Data size: 491654 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ctinyint (type: tinyint), cbigint (type: bigint), cint (type: int), cfloat (type: float), UDFToDouble(cint) (type: double), (UDFToDouble(cint) * UDFToDouble(cint)) (type: double), UDFToDouble(cbigint) (type: double), (UDFToDouble(cbigint) * UDFToDouble(cbigint)) (type: double) diff --git ql/src/test/results/clientpositive/mapjoin47.q.out ql/src/test/results/clientpositive/mapjoin47.q.out index 294dd69de5c..d3e61f8e883 100644 --- ql/src/test/results/clientpositive/mapjoin47.q.out +++ ql/src/test/results/clientpositive/mapjoin47.q.out @@ -748,8 +748,8 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3 - residual filter predicates: {(struct(_col0,_col2)) IN (const struct(100,100), const struct(101,101), const struct(102,102))} - Statistics: Num rows: 9375 Data size: 180600 Basic stats: COMPLETE Column stats: NONE + residual filter predicates: {(struct(UDFToDouble(_col0),UDFToDouble(_col2))) IN (const struct(100.0D,100.0D), const struct(101.0D,101.0D), const struct(102.0D,102.0D))} + Statistics: Num rows: 6250 Data size: 120400 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 190 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/parquet_vectorization_13.q.out ql/src/test/results/clientpositive/parquet_vectorization_13.q.out index 0efce98b555..0bd81bb886c 100644 --- ql/src/test/results/clientpositive/parquet_vectorization_13.q.out +++ ql/src/test/results/clientpositive/parquet_vectorization_13.q.out @@ -83,7 +83,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -92,7 +92,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28789.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28788.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 2730 Data size: 32760 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) @@ -405,7 +405,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -414,7 +414,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28801.388)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28801.336)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 2730 Data size: 32760 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) diff --git ql/src/test/results/clientpositive/parquet_vectorization_6.q.out ql/src/test/results/clientpositive/parquet_vectorization_6.q.out index 0bb68883640..b35c11eaf16 100644 --- ql/src/test/results/clientpositive/parquet_vectorization_6.q.out +++ ql/src/test/results/clientpositive/parquet_vectorization_6.q.out @@ -58,7 +58,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257))))) (type: boolean) + filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0))))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -67,7 +67,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprAndExpr(children: FilterLongColNotEqualLongScalar(col 0:tinyint, val 0), FilterExprOrExpr(children: FilterExprAndExpr(children: FilterLongColLessEqualLongScalar(col 10:boolean, val 0), FilterLongColGreaterEqualLongColumn(col 11:boolean, col 10:boolean)), FilterExprAndExpr(children: SelectColumnIsNotNull(col 3:bigint), FilterExprOrExpr(children: FilterStringColLikeStringScalar(col 7:string, pattern %a), FilterDoubleColLessEqualDoubleScalar(col 4:float, val -257.0))))) - predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257)))) and (ctinyint <> 0Y)) (type: boolean) + predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0)))) and (ctinyint <> 0Y)) (type: boolean) Statistics: Num rows: 11605 Data size: 139260 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), cfloat (type: float), cstring1 (type: string), (988888 * UDFToInteger(csmallint)) (type: int), (- csmallint) (type: smallint), (- cfloat) (type: float), (-26.28D / UDFToDouble(cfloat)) (type: double), (cfloat * 359.0) (type: float), (cint % UDFToInteger(ctinyint)) (type: int), (- cdouble) (type: double), (UDFToInteger(ctinyint) - -75) (type: int), (762 * (cint % UDFToInteger(ctinyint))) (type: int) diff --git ql/src/test/results/clientpositive/parquet_vectorization_8.q.out ql/src/test/results/clientpositive/parquet_vectorization_8.q.out index 957bd7b264c..e01cf4a8c0f 100644 --- ql/src/test/results/clientpositive/parquet_vectorization_8.q.out +++ ql/src/test/results/clientpositive/parquet_vectorization_8.q.out @@ -66,7 +66,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D)) or (cfloat < -6432) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) + filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D)) or (cfloat < -6432.0) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -75,7 +75,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 10.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 16.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0))) - predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D))) (type: boolean) + predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432.0) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) @@ -272,7 +272,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D)) or (cfloat < -6432) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) + filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D)) or (cfloat < -6432.0) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -281,7 +281,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 12.503)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 11.998)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0))) - predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D))) (type: boolean) + predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432.0) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) diff --git ql/src/test/results/clientpositive/pcs.q.out ql/src/test/results/clientpositive/pcs.q.out index 4a89fb0787e..225624eb978 100644 --- ql/src/test/results/clientpositive/pcs.q.out +++ ql/src/test/results/clientpositive/pcs.q.out @@ -299,12 +299,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: pcs_t1 - filterExpr: ((struct(ds,key)) IN (const struct('2000-04-08',1), const struct('2000-04-09',2)) and (struct(ds)) IN (struct('2000-04-08'), struct('2000-04-09'))) (type: boolean) + filterExpr: ((struct(key,ds)) IN (const struct(1,'2000-04-08'), const struct(2,'2000-04-09')) and (struct(ds)) IN (struct('2000-04-08'), struct('2000-04-09'))) (type: boolean) Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (struct(ds,key)) IN (const struct('2000-04-08',1), const struct('2000-04-09',2)) (type: boolean) + predicate: (struct(key,ds)) IN (const struct(1,'2000-04-08'), const struct(2,'2000-04-09')) (type: boolean) Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -655,32 +655,44 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - filterExpr: ((struct(ds)) IN (const struct('2000-04-08'), const struct('2000-04-09')) and (struct(ds)) IN (const struct('2000-04-09'), const struct('2000-04-08'))) (type: boolean) + filterExpr: ((key) IN (1, 2) and (ds) IN ('2000-04-08', '2000-04-09')) (type: boolean) Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: ds (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: ds (type: string) + Filter Operator + isSamplingPred: false + predicate: (key) IN (1, 2) (type: boolean) Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE - tag: 0 - value expressions: key (type: int) - auto parallelism: false + Select Operator + expressions: key (type: int), ds (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: _col0 (type: int) + auto parallelism: false TableScan alias: b - filterExpr: ((struct(ds)) IN (const struct('2000-04-09'), const struct('2000-04-08')) and (struct(ds)) IN (const struct('2000-04-08'), const struct('2000-04-09'))) (type: boolean) + filterExpr: (ds) IN ('2000-04-09', '2000-04-08') (type: boolean) Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: ds (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: ds (type: string) + Select Operator + expressions: key (type: int), ds (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE - tag: 1 - value expressions: key (type: int) - auto parallelism: false + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: _col0 (type: int) + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -781,24 +793,24 @@ STAGE PLANS: name: default.pcs_t1 name: default.pcs_t1 Truncated Path -> Alias: - /pcs_t1/ds=2000-04-08 [a, b] - /pcs_t1/ds=2000-04-09 [a, b] + /pcs_t1/ds=2000-04-08 [$hdt$_0:a, $hdt$_1:b] + /pcs_t1/ds=2000-04-09 [$hdt$_0:a, $hdt$_1:b] Needs Tagging: true Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 ds (type: string) - 1 ds (type: string) - outputColumnNames: _col0, _col2, _col6, _col8 + 0 _col1 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 44 Data size: 352 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false - predicate: (struct(_col2,_col0,_col8)) IN (const struct('2000-04-08',1,'2000-04-09'), const struct('2000-04-09',2,'2000-04-08')) (type: boolean) + predicate: (struct(_col0,_col1,_col3)) IN (const struct(1,'2000-04-08','2000-04-09'), const struct(2,'2000-04-09','2000-04-08')) (type: boolean) Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: string), _col6 (type: int) + expressions: _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -864,7 +876,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: pcs_t1 - filterExpr: ((struct(ds,(key + key))) IN (const struct('2000-04-08',1), const struct('2000-04-09',2)) and (struct(ds)) IN (const struct('2000-04-08'), const struct('2000-04-09'))) (type: boolean) + filterExpr: ((struct(ds,(key + key))) IN (const struct('2000-04-08',1), const struct('2000-04-09',2)) and (struct(ds)) IN (struct('2000-04-08'), struct('2000-04-09'))) (type: boolean) Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator @@ -1043,10 +1055,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: pcs_t1 - filterExpr: ((struct(ds,key)) IN (const struct('2000-04-08',1), const struct('2000-04-09',2)) and (struct(ds)) IN (const struct('2000-04-08'), const struct('2000-04-09'))) (type: boolean) + filterExpr: ((struct(key,ds)) IN (const struct(1,'2000-04-08'), const struct(2,'2000-04-09')) and (struct(ds)) IN (struct('2000-04-08'), struct('2000-04-09'))) (type: boolean) Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (struct(ds,key)) IN (const struct('2000-04-08',1), const struct('2000-04-09',2)) (type: boolean) + predicate: (struct(key,ds)) IN (const struct(1,'2000-04-08'), const struct(2,'2000-04-09')) (type: boolean) Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) @@ -1260,6 +1272,9 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@pcs_t1 POSTHOOK: Input: default@pcs_t1@ds=2000-04-08 #### A masked pattern was here #### +OPTIMIZED SQL: SELECT CAST('2000-04-08' AS STRING) AS `$f0` +FROM `default`.`pcs_t1` +WHERE `ds` = '2000-04-08' STAGE DEPENDENCIES: Stage-0 is a root stage @@ -1317,18 +1332,14 @@ STAGE PLANS: Processor Tree: TableScan alias: pcs_t1 - filterExpr: (const struct(10)) IN (const struct(10), const struct(11)) (type: boolean) + filterExpr: (ds = '2000-04-08') (type: boolean) Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (const struct(10)) IN (const struct(10), const struct(11)) (type: boolean) - Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE - ListSink + Select Operator + expressions: '2000-04-08' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE + ListSink PREHOOK: query: select ds from pcs_t1 where struct(case when ds='2000-04-08' then 10 else 20 end) in (struct(10),struct(11)) PREHOOK: type: QUERY @@ -1386,7 +1397,7 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (struct(ds,key,rand(100))) IN (const struct('2000-04-08',1,0.2), const struct('2000-04-09',2,0.3)) (type: boolean) + predicate: (struct(ds,key,rand(100))) IN (const struct('2000-04-08',1,0.2D), const struct('2000-04-09',2,0.3D)) (type: boolean) Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -1414,6 +1425,7 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false + Execution mode: vectorized Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1547,12 +1559,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: pcs_t1 - filterExpr: (struct(((ds = '2000-04-08') or (key = 2)),key)) IN (const struct(true,2), const struct(false,3)) (type: boolean) + filterExpr: (struct(key,((ds = '2000-04-08') or (key = 2)))) IN (const struct(2,true), const struct(3,false)) (type: boolean) Statistics: Num rows: 60 Data size: 480 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (struct(((ds = '2000-04-08') or (key = 2)),key)) IN (const struct(true,2), const struct(false,3)) (type: boolean) + predicate: (struct(key,((ds = '2000-04-08') or (key = 2)))) IN (const struct(2,true), const struct(3,false)) (type: boolean) Statistics: Num rows: 30 Data size: 240 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -1779,12 +1791,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: pcs_t1 - filterExpr: ((key = 3) or ((struct(((ds = '2000-04-08') or (key = 2)),key)) IN (const struct(true,2), const struct(false,3)) and ((key + 5) > 0))) (type: boolean) + filterExpr: ((key = 3) or ((struct(key,((ds = '2000-04-08') or (key = 2)))) IN (const struct(2,true), const struct(3,false)) and ((key + 5) > 0))) (type: boolean) Statistics: Num rows: 60 Data size: 480 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (((struct(((ds = '2000-04-08') or (key = 2)),key)) IN (const struct(true,2), const struct(false,3)) and ((key + 5) > 0)) or (key = 3)) (type: boolean) + predicate: (((struct(key,((ds = '2000-04-08') or (key = 2)))) IN (const struct(2,true), const struct(3,false)) and ((key + 5) > 0)) or (key = 3)) (type: boolean) Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) diff --git ql/src/test/results/clientpositive/perf/tez/cbo_query15.q.out ql/src/test/results/clientpositive/perf/tez/cbo_query15.q.out index 02ad7c207c0..522bbb031b1 100644 --- ql/src/test/results/clientpositive/perf/tez/cbo_query15.q.out +++ ql/src/test/results/clientpositive/perf/tez/cbo_query15.q.out @@ -50,7 +50,7 @@ CBO PLAN: HiveSortLimit(sort0=[$0], dir0=[ASC], fetch=[100]) HiveProject(ca_zip=[$0], $f1=[$1]) HiveAggregate(group=[{4}], agg#0=[sum($7)]) - HiveJoin(condition=[AND(=($6, $0), OR(IN(substr($4, 1, 5), _UTF-16LE'85669', _UTF-16LE'86197', _UTF-16LE'88274', _UTF-16LE'83405', _UTF-16LE'86475', _UTF-16LE'85392', _UTF-16LE'85460', _UTF-16LE'80348', _UTF-16LE'81792'), >($7, 500), IN($3, _UTF-16LE'CA', _UTF-16LE'WA', _UTF-16LE'GA')))], joinType=[inner], algorithm=[none], cost=[not available]) + HiveJoin(condition=[AND(=($6, $0), OR(>($7, 500), IN(substr($4, 1, 5), _UTF-16LE'85669', _UTF-16LE'86197', _UTF-16LE'88274', _UTF-16LE'83405', _UTF-16LE'86475', _UTF-16LE'85392', _UTF-16LE'85460', _UTF-16LE'80348', _UTF-16LE'81792'), IN($3, _UTF-16LE'CA', _UTF-16LE'WA', _UTF-16LE'GA')))], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($1, $2)], joinType=[inner], algorithm=[none], cost=[not available]) HiveProject(c_customer_sk=[$0], c_current_addr_sk=[$4]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($4))]) diff --git ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query15.q.out ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query15.q.out index b260731c8f9..32404b6209c 100644 --- ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query15.q.out +++ ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query15.q.out @@ -50,7 +50,7 @@ CBO PLAN: HiveSortLimit(sort0=[$0], dir0=[ASC], fetch=[100]) HiveProject(ca_zip=[$0], $f1=[$1]) HiveAggregate(group=[{3}], agg#0=[sum($8)]) - HiveJoin(condition=[AND(=($7, $0), OR($4, $9, $5))], joinType=[inner], algorithm=[none], cost=[not available]) + HiveJoin(condition=[AND(=($7, $0), OR($9, $4, $5))], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($1, $2)], joinType=[inner], algorithm=[none], cost=[not available]) HiveProject(c_customer_sk=[$0], c_current_addr_sk=[$4]) HiveFilter(condition=[IS NOT NULL($4)]) diff --git ql/src/test/results/clientpositive/ppd_udf_col.q.out ql/src/test/results/clientpositive/ppd_udf_col.q.out index 814fb5afcfd..9ef340d512b 100644 --- ql/src/test/results/clientpositive/ppd_udf_col.q.out +++ ql/src/test/results/clientpositive/ppd_udf_col.q.out @@ -432,7 +432,7 @@ STAGE PLANS: outputColumnNames: _col0, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col2 <= 0.1) (type: boolean) + predicate: (_col2 <= 0.1D) (type: boolean) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: double) @@ -491,7 +491,7 @@ STAGE PLANS: outputColumnNames: _col0, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col2 <= 0.1) and (_col2 > 0.1)) (type: boolean) + predicate: ((_col2 <= 0.1D) and (_col2 > 0.1D)) (type: boolean) Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: double) @@ -644,7 +644,7 @@ STAGE PLANS: outputColumnNames: _col0, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col2 <= 0.1) (type: boolean) + predicate: (_col2 <= 0.1D) (type: boolean) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: double) @@ -703,14 +703,14 @@ STAGE PLANS: outputColumnNames: _col0, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col2 <= 0.1) (type: boolean) + predicate: (_col2 <= 0.1D) (type: boolean) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col1 > 0.1) (type: boolean) + predicate: (_col1 > 0.1D) (type: boolean) Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 20 diff --git ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out index e796443d962..074b56dbc8b 100644 --- ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out +++ ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out @@ -60,10 +60,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src - filterExpr: (((value) IN ('val_400', 'val_500') and (key) IN (400, 450)) or ((value) IN ('val_100', 'val_200', 'val_300') and (key) IN (100, 150, 200))) (type: boolean) + filterExpr: ((((value = 'val_400') or (value = 'val_500')) and ((key = 400) or (key = 450))) or (((value = 'val_100') or (value = 'val_200') or (value = 'val_300')) and ((key = 100) or (key = 150) or (key = 200)))) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((value) IN ('val_100', 'val_200', 'val_300') and (key) IN (100, 150, 200)) or ((value) IN ('val_400', 'val_500') and (key) IN (400, 450))) (type: boolean) + predicate: ((((value = 'val_100') or (value = 'val_200') or (value = 'val_300')) and ((key = 100) or (key = 150) or (key = 200))) or (((value = 'val_400') or (value = 'val_500')) and ((key = 400) or (key = 450)))) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -71,12 +71,13 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) + Execution mode: vectorized Reducer 2 Reduce Operator Tree: Forward Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((KEY._col0) IN (100, 150, 200) and (VALUE._col0) IN ('val_100', 'val_200', 'val_300')) (type: boolean) + predicate: (((KEY._col0 = 100) or (KEY._col0 = 150) or (KEY._col0 = 200)) and ((VALUE._col0 = 'val_100') or (VALUE._col0 = 'val_200') or (VALUE._col0 = 'val_300'))) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -97,7 +98,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.e1_n1 Filter Operator - predicate: ((KEY._col0) IN (400, 450) and (VALUE._col0) IN ('val_400', 'val_500')) (type: boolean) + predicate: (((KEY._col0 = 400) or (KEY._col0 = 450)) and ((VALUE._col0 = 'val_400') or (VALUE._col0 = 'val_500'))) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -425,7 +426,7 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((value) IN ('val_100', 'val_200', 'val_300') and (key) IN (100, 150, 200)) or ((value) IN ('val_400', 'val_500') and (key) IN (400, 450))) (type: boolean) + predicate: ((((value = 'val_100') or (value = 'val_200') or (value = 'val_300')) and ((key = 100) or (key = 150) or (key = 200))) or (((value = 'val_400') or (value = 'val_500')) and ((key = 400) or (key = 450)))) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -433,12 +434,13 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) + Execution mode: vectorized Reducer 2 Reduce Operator Tree: Forward Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((KEY._col0) IN (100, 150, 200) and (VALUE._col0) IN ('val_100', 'val_200', 'val_300')) (type: boolean) + predicate: (((KEY._col0 = 100) or (KEY._col0 = 150) or (KEY._col0 = 200)) and ((VALUE._col0 = 'val_100') or (VALUE._col0 = 'val_200') or (VALUE._col0 = 'val_300'))) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -459,7 +461,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.e1_n1 Filter Operator - predicate: ((KEY._col0) IN (400, 450) and (VALUE._col0) IN ('val_400', 'val_500')) (type: boolean) + predicate: (((KEY._col0 = 400) or (KEY._col0 = 450)) and ((VALUE._col0 = 'val_400') or (VALUE._col0 = 'val_500'))) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_13.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_13.q.out index 38122393439..19880a820ed 100644 --- ql/src/test/results/clientpositive/spark/parquet_vectorization_13.q.out +++ ql/src/test/results/clientpositive/spark/parquet_vectorization_13.q.out @@ -88,7 +88,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -97,7 +97,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28789.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28788.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 2730 Data size: 32760 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) @@ -421,7 +421,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -430,7 +430,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28801.388)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28801.336)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 2730 Data size: 32760 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_6.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_6.q.out index 6108457aad3..eef9f39e264 100644 --- ql/src/test/results/clientpositive/spark/parquet_vectorization_6.q.out +++ ql/src/test/results/clientpositive/spark/parquet_vectorization_6.q.out @@ -61,7 +61,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257))))) (type: boolean) + filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0))))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -70,7 +70,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprAndExpr(children: FilterLongColNotEqualLongScalar(col 0:tinyint, val 0), FilterExprOrExpr(children: FilterExprAndExpr(children: FilterLongColLessEqualLongScalar(col 10:boolean, val 0), FilterLongColGreaterEqualLongColumn(col 11:boolean, col 10:boolean)), FilterExprAndExpr(children: SelectColumnIsNotNull(col 3:bigint), FilterExprOrExpr(children: FilterStringColLikeStringScalar(col 7:string, pattern %a), FilterDoubleColLessEqualDoubleScalar(col 4:float, val -257.0))))) - predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257)))) and (ctinyint <> 0Y)) (type: boolean) + predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0)))) and (ctinyint <> 0Y)) (type: boolean) Statistics: Num rows: 11605 Data size: 139260 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), cfloat (type: float), cstring1 (type: string), (988888 * UDFToInteger(csmallint)) (type: int), (- csmallint) (type: smallint), (- cfloat) (type: float), (-26.28D / UDFToDouble(cfloat)) (type: double), (cfloat * 359.0) (type: float), (cint % UDFToInteger(ctinyint)) (type: int), (- cdouble) (type: double), (UDFToInteger(ctinyint) - -75) (type: int), (762 * (cint % UDFToInteger(ctinyint))) (type: int) diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_8.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_8.q.out index 3352dedc584..657e05cb097 100644 --- ql/src/test/results/clientpositive/spark/parquet_vectorization_8.q.out +++ ql/src/test/results/clientpositive/spark/parquet_vectorization_8.q.out @@ -71,7 +71,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D)) or (cfloat < -6432) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) + filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D)) or (cfloat < -6432.0) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -80,7 +80,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 10.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 16.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0))) - predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D))) (type: boolean) + predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432.0) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) @@ -295,7 +295,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D)) or (cfloat < -6432) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) + filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D)) or (cfloat < -6432.0) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -304,7 +304,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 12.503)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 11.998)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0))) - predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D))) (type: boolean) + predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432.0) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) diff --git ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out index f5a4c9ad861..7477696c4e4 100644 --- ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out +++ ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out @@ -452,7 +452,7 @@ Stage-0 Group By Operator [GBY_6] (rows=2 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_33] (rows=5 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 8 [PARTITION-LEVEL SORT] @@ -468,7 +468,7 @@ Stage-0 Group By Operator [GBY_13] (rows=2 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_34] (rows=5 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -544,7 +544,7 @@ Stage-0 Group By Operator [GBY_6] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_35] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 8 [PARTITION-LEVEL SORT] @@ -560,7 +560,7 @@ Stage-0 Group By Operator [GBY_13] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_36] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -629,7 +629,7 @@ Stage-0 Group By Operator [GBY_6] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_30] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 7 [PARTITION-LEVEL SORT] @@ -645,7 +645,7 @@ Stage-0 Group By Operator [GBY_13] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_31] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -719,7 +719,7 @@ Stage-0 Group By Operator [GBY_6] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_34] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 8 [PARTITION-LEVEL SORT] @@ -735,7 +735,7 @@ Stage-0 Group By Operator [GBY_13] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_35] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -804,7 +804,7 @@ Stage-0 Group By Operator [GBY_6] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_30] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 7 [PARTITION-LEVEL SORT] @@ -820,7 +820,7 @@ Stage-0 Group By Operator [GBY_13] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_31] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -1263,7 +1263,7 @@ Stage-0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1","_col2"] Filter Operator [FIL_17] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 3 [PARTITION-LEVEL SORT] @@ -1272,7 +1272,7 @@ Stage-0 Select Operator [SEL_5] (rows=9 width=89) Output:["_col0","_col1"] Filter Operator [FIL_18] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 4 [PARTITION-LEVEL SORT] @@ -1324,7 +1324,7 @@ Stage-0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1","_col2"] Filter Operator [FIL_17] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 3 [PARTITION-LEVEL SORT] @@ -1333,7 +1333,7 @@ Stage-0 Select Operator [SEL_5] (rows=9 width=89) Output:["_col0","_col1"] Filter Operator [FIL_18] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 4 [PARTITION-LEVEL SORT] @@ -1590,7 +1590,7 @@ Stage-0 Group By Operator [GBY_3] (rows=3 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_45] (rows=6 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0))) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) TableScan [TS_0] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 9 [PARTITION-LEVEL SORT] @@ -1614,7 +1614,7 @@ Stage-0 Group By Operator [GBY_16] (rows=3 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_46] (rows=6 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0))) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) TableScan [TS_13] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -1653,7 +1653,7 @@ Stage-0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1"] Filter Operator [FIL_13] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 3 [PARTITION-LEVEL SORT] @@ -1705,7 +1705,7 @@ Stage-0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1","_col2"] Filter Operator [FIL_19] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 3 [PARTITION-LEVEL SORT] @@ -1716,7 +1716,7 @@ Stage-0 Select Operator [SEL_5] (rows=9 width=85) Output:["_col0"] Filter Operator [FIL_20] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 4 [PARTITION-LEVEL SORT] @@ -1805,7 +1805,7 @@ Stage-0 Group By Operator [GBY_3] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_35] (rows=1 width=93) - predicate:((((c_int + 1) + 1) >= 0) and (((c_int + 1) > 0) or (UDFToDouble(key) >= 0.0D)) and ((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (UDFToDouble(key) > 0.0D) and (c_float > 0)) + predicate:((((c_int + 1) + 1) >= 0) and (((c_int + 1) > 0) or (UDFToDouble(key) >= 0.0D)) and ((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (UDFToDouble(key) > 0.0D) and (c_float > 0.0)) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 7 [PARTITION-LEVEL SORT] @@ -1823,7 +1823,7 @@ Stage-0 Group By Operator [GBY_12] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_36] (rows=1 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (UDFToDouble(key) > 0.0D) and (c_float > 0)) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (UDFToDouble(key) > 0.0D) and (c_float > 0.0)) TableScan [TS_9] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] diff --git ql/src/test/results/clientpositive/spark/vector_between_in.q.out ql/src/test/results/clientpositive/spark/vector_between_in.q.out index da2fbe741fa..a93239bf981 100644 --- ql/src/test/results/clientpositive/spark/vector_between_in.q.out +++ ql/src/test/results/clientpositive/spark/vector_between_in.q.out @@ -1127,13 +1127,13 @@ STAGE PLANS: TableScan Vectorization: native: true Select Operator - expressions: ((cdate = DATE'1969-10-26') or (cdate = DATE'1969-07-14')) (type: boolean) + expressions: (cdate) IN (DATE'1969-10-26', DATE'1969-07-14') (type: boolean) outputColumnNames: _col0 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [7] - selectExpressions: ColOrCol(col 5:boolean, col 6:boolean)(children: DateColEqualDateScalar(col 3:date, date 1969-10-26) -> 5:boolean, DateColEqualDateScalar(col 3:date, date 1969-07-14) -> 6:boolean) -> 7:boolean + projectedOutputColumnNums: [5] + selectExpressions: LongColumnInList(col 3, values [-67, -171]) -> 5:boolean Statistics: Num rows: 12289 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1141,7 +1141,7 @@ STAGE PLANS: aggregators: VectorUDAFCountStar(*) -> bigint className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 7:boolean + keyExpressions: col 5:boolean native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -1267,13 +1267,13 @@ STAGE PLANS: TableScan Vectorization: native: true Select Operator - expressions: ((cdecimal1 = 2365.8945945946) or (cdecimal1 = 881.0135135135) or (cdecimal1 = -3367.6517567568)) (type: boolean) + expressions: (cdecimal1) IN (2365.8945945946, 881.0135135135, -3367.6517567568) (type: boolean) outputColumnNames: _col0 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [8] - selectExpressions: VectorUDFAdaptor(((cdecimal1 = 2365.8945945946) or (cdecimal1 = 881.0135135135) or (cdecimal1 = -3367.6517567568)))(children: DecimalColEqualDecimalScalar(col 1:decimal(20,10), val 2365.8945945946) -> 5:boolean, DecimalColEqualDecimalScalar(col 1:decimal(20,10), val 881.0135135135) -> 6:boolean, DecimalColEqualDecimalScalar(col 1:decimal(20,10), val -3367.6517567568) -> 7:boolean) -> 8:boolean + projectedOutputColumnNums: [5] + selectExpressions: DecimalColumnInList(col 1:decimal(20,10), values [2365.8945945946, 881.0135135135, -3367.6517567568]) -> 5:boolean Statistics: Num rows: 12289 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1281,7 +1281,7 @@ STAGE PLANS: aggregators: VectorUDAFCountStar(*) -> bigint className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 8:boolean + keyExpressions: col 5:boolean native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -1307,7 +1307,7 @@ STAGE PLANS: featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reducer 2 Execution mode: vectorized diff --git ql/src/test/results/clientpositive/spark/vectorization_13.q.out ql/src/test/results/clientpositive/spark/vectorization_13.q.out index 34ec9c42dd3..c25649bd145 100644 --- ql/src/test/results/clientpositive/spark/vectorization_13.q.out +++ ql/src/test/results/clientpositive/spark/vectorization_13.q.out @@ -88,7 +88,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -98,7 +98,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28789.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28788.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 2730 Data size: 646063 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) @@ -445,7 +445,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -454,7 +454,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28801.388)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28801.336)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 2730 Data size: 646063 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) diff --git ql/src/test/results/clientpositive/spark/vectorization_6.q.out ql/src/test/results/clientpositive/spark/vectorization_6.q.out index 5679bb8cfa4..d1cc6e39eda 100644 --- ql/src/test/results/clientpositive/spark/vectorization_6.q.out +++ ql/src/test/results/clientpositive/spark/vectorization_6.q.out @@ -61,7 +61,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257))))) (type: boolean) + filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0))))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -71,7 +71,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprAndExpr(children: FilterLongColNotEqualLongScalar(col 0:tinyint, val 0), FilterExprOrExpr(children: FilterExprAndExpr(children: FilterLongColLessEqualLongScalar(col 10:boolean, val 0), FilterLongColGreaterEqualLongColumn(col 11:boolean, col 10:boolean)), FilterExprAndExpr(children: SelectColumnIsNotNull(col 3:bigint), FilterExprOrExpr(children: FilterStringColLikeStringScalar(col 7:string, pattern %a), FilterDoubleColLessEqualDoubleScalar(col 4:float, val -257.0))))) - predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257)))) and (ctinyint <> 0Y)) (type: boolean) + predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0)))) and (ctinyint <> 0Y)) (type: boolean) Statistics: Num rows: 11605 Data size: 2746359 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), cfloat (type: float), cstring1 (type: string), (988888 * UDFToInteger(csmallint)) (type: int), (- csmallint) (type: smallint), (- cfloat) (type: float), (-26.28D / UDFToDouble(cfloat)) (type: double), (cfloat * 359.0) (type: float), (cint % UDFToInteger(ctinyint)) (type: int), (- cdouble) (type: double), (UDFToInteger(ctinyint) - -75) (type: int), (762 * (cint % UDFToInteger(ctinyint))) (type: int) diff --git ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out index 231dea6de3e..6c3ab0d8519 100644 --- ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out +++ ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out @@ -617,7 +617,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((ctimestamp1 = ctimestamp2) or (cfloat = 762) or (cstring1 = 'ss') or ((UDFToLong(csmallint) <= cbigint) and (cboolean2 = 1)) or (cboolean1 is not null and ctimestamp2 is not null and (cstring2 > 'a'))) (type: boolean) + filterExpr: ((ctimestamp1 = ctimestamp2) or (cfloat = 762.0) or (cstring1 = 'ss') or ((UDFToLong(csmallint) <= cbigint) and (cboolean2 = 1)) or (cboolean1 is not null and ctimestamp2 is not null and (cstring2 > 'a'))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -626,7 +626,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterTimestampColEqualTimestampColumn(col 8:timestamp, col 9:timestamp), FilterDoubleColEqualDoubleScalar(col 4:float, val 762.0), FilterStringGroupColEqualStringScalar(col 6:string, val ss), FilterExprAndExpr(children: FilterLongColLessEqualLongColumn(col 1:bigint, col 3:bigint)(children: col 1:smallint), FilterLongColEqualLongScalar(col 11:boolean, val 1)), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), SelectColumnIsNotNull(col 9:timestamp), FilterStringGroupColGreaterStringScalar(col 7:string, val a))) - predicate: (((UDFToLong(csmallint) <= cbigint) and (cboolean2 = 1)) or (cboolean1 is not null and ctimestamp2 is not null and (cstring2 > 'a')) or (cfloat = 762) or (cstring1 = 'ss') or (ctimestamp1 = ctimestamp2)) (type: boolean) + predicate: (((UDFToLong(csmallint) <= cbigint) and (cboolean2 = 1)) or (cboolean1 is not null and ctimestamp2 is not null and (cstring2 > 'a')) or (cfloat = 762.0) or (cstring1 = 'ss') or (ctimestamp1 = ctimestamp2)) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cbigint (type: bigint), ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cdouble (type: double), UDFToDouble(cbigint) (type: double), (UDFToDouble(cbigint) * UDFToDouble(cbigint)) (type: double), UDFToDouble(csmallint) (type: double), (UDFToDouble(csmallint) * UDFToDouble(csmallint)) (type: double), (cdouble * cdouble) (type: double) @@ -853,7 +853,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: (((ctimestamp2 <= ctimestamp1) and (UDFToDouble(cbigint) <> cdouble) and (cstring1 >= 'ss')) or ((csmallint < UDFToShort(ctinyint)) and (UDFToDouble(ctimestamp1) >= 0.0D)) or (cfloat = 17)) (type: boolean) + filterExpr: (((ctimestamp2 <= ctimestamp1) and (UDFToDouble(cbigint) <> cdouble) and (cstring1 >= 'ss')) or ((csmallint < UDFToShort(ctinyint)) and (UDFToDouble(ctimestamp1) >= 0.0D)) or (cfloat = 17.0)) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -862,7 +862,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterTimestampColLessEqualTimestampColumn(col 9:timestamp, col 8:timestamp), FilterDoubleColNotEqualDoubleColumn(col 13:double, col 5:double)(children: CastLongToDouble(col 3:bigint) -> 13:double), FilterStringGroupColGreaterEqualStringScalar(col 6:string, val ss)), FilterExprAndExpr(children: FilterLongColLessLongColumn(col 1:smallint, col 0:smallint)(children: col 0:tinyint), FilterDoubleColGreaterEqualDoubleScalar(col 13:double, val 0.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double)), FilterDoubleColEqualDoubleScalar(col 4:float, val 17.0)) - predicate: (((csmallint < UDFToShort(ctinyint)) and (UDFToDouble(ctimestamp1) >= 0.0D)) or ((ctimestamp2 <= ctimestamp1) and (UDFToDouble(cbigint) <> cdouble) and (cstring1 >= 'ss')) or (cfloat = 17)) (type: boolean) + predicate: (((csmallint < UDFToShort(ctinyint)) and (UDFToDouble(ctimestamp1) >= 0.0D)) or ((ctimestamp2 <= ctimestamp1) and (UDFToDouble(cbigint) <> cdouble) and (cstring1 >= 'ss')) or (cfloat = 17.0)) (type: boolean) Statistics: Num rows: 8874 Data size: 2100060 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctinyint (type: tinyint), cbigint (type: bigint), cint (type: int), cfloat (type: float), UDFToDouble(cint) (type: double), (UDFToDouble(cint) * UDFToDouble(cint)) (type: double), UDFToDouble(cbigint) (type: double), (UDFToDouble(cbigint) * UDFToDouble(cbigint)) (type: double) diff --git ql/src/test/results/clientpositive/vector_non_constant_in_expr.q.out ql/src/test/results/clientpositive/vector_non_constant_in_expr.q.out index e8e5c062783..23f6d0a0066 100644 --- ql/src/test/results/clientpositive/vector_non_constant_in_expr.q.out +++ ql/src/test/results/clientpositive/vector_non_constant_in_expr.q.out @@ -20,10 +20,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((cint = UDFToInteger(ctinyint)) or (cint = UDFToInteger(cbigint))) (type: boolean) + filterExpr: ((cint = UDFToInteger(ctinyint)) or (UDFToLong(cint) = cbigint)) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((cint = UDFToInteger(cbigint)) or (cint = UDFToInteger(ctinyint))) (type: boolean) + predicate: ((UDFToLong(cint) = cbigint) or (cint = UDFToInteger(ctinyint))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) diff --git ql/src/test/results/clientpositive/vector_struct_in.q.out ql/src/test/results/clientpositive/vector_struct_in.q.out index 546ef02fa42..aec7ac433d4 100644 --- ql/src/test/results/clientpositive/vector_struct_in.q.out +++ ql/src/test/results/clientpositive/vector_struct_in.q.out @@ -842,7 +842,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test_4 - filterExpr: (struct(my_bigint,my_string,my_double)) IN (const struct(1L,'a',1.5D), const struct(1L,'b',-0.5D), const struct(3L,'b',1.5D), const struct(1L,'d',1.5D), const struct(1L,'c',1.5D), const struct(1L,'b',2.5D), const struct(1L,'b',0.5D), const struct(5L,'b',1.5D), const struct(1L,'a',0.5D), const struct(3L,'b',1.5D)) (type: boolean) + filterExpr: (struct(my_bigint,my_string,my_double)) IN (const struct(1L,'a',1.5D), const struct(1L,'b',-0.5D), const struct(3L,'b',1.5D), const struct(1L,'d',1.5D), const struct(1L,'c',1.5D), const struct(1L,'b',2.5D), const struct(1L,'b',0.5D), const struct(5L,'b',1.5D), const struct(1L,'a',0.5D)) (type: boolean) Statistics: Num rows: 3 Data size: 303 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -851,7 +851,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterStructColumnInList(structExpressions [col 0:bigint, col 1:string, col 2:double], fieldVectorColumnTypes [LONG, BYTES, DOUBLE], structColumnMap [0, 1, 2]) - predicate: (struct(my_bigint,my_string,my_double)) IN (const struct(1L,'a',1.5D), const struct(1L,'b',-0.5D), const struct(3L,'b',1.5D), const struct(1L,'d',1.5D), const struct(1L,'c',1.5D), const struct(1L,'b',2.5D), const struct(1L,'b',0.5D), const struct(5L,'b',1.5D), const struct(1L,'a',0.5D), const struct(3L,'b',1.5D)) (type: boolean) + predicate: (struct(my_bigint,my_string,my_double)) IN (const struct(1L,'a',1.5D), const struct(1L,'b',-0.5D), const struct(3L,'b',1.5D), const struct(1L,'d',1.5D), const struct(1L,'c',1.5D), const struct(1L,'b',2.5D), const struct(1L,'b',0.5D), const struct(5L,'b',1.5D), const struct(1L,'a',0.5D)) (type: boolean) Statistics: Num rows: 3 Data size: 303 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: my_bigint (type: bigint), my_string (type: string), my_double (type: double) @@ -973,7 +973,7 @@ STAGE PLANS: TableScan Vectorization: native: true Select Operator - expressions: my_bigint (type: bigint), my_string (type: string), my_double (type: double), (struct(my_bigint,my_string,my_double)) IN (const struct(1L,'a',1.5D), const struct(1L,'b',-0.5D), const struct(3L,'b',1.5D), const struct(1L,'d',1.5D), const struct(1L,'c',1.5D), const struct(1L,'b',2.5D), const struct(1L,'b',0.5D), const struct(5L,'b',1.5D), const struct(1L,'a',0.5D), const struct(3L,'b',1.5D)) (type: boolean) + expressions: my_bigint (type: bigint), my_string (type: string), my_double (type: double), (struct(my_bigint,my_string,my_double)) IN (const struct(1L,'a',1.5D), const struct(1L,'b',-0.5D), const struct(3L,'b',1.5D), const struct(1L,'d',1.5D), const struct(1L,'c',1.5D), const struct(1L,'b',2.5D), const struct(1L,'b',0.5D), const struct(5L,'b',1.5D), const struct(1L,'a',0.5D)) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3 Select Vectorization: className: VectorSelectOperator diff --git ql/src/test/results/clientpositive/vectorization_13.q.out ql/src/test/results/clientpositive/vectorization_13.q.out index 8897f8427ff..2ed66189e7c 100644 --- ql/src/test/results/clientpositive/vectorization_13.q.out +++ ql/src/test/results/clientpositive/vectorization_13.q.out @@ -83,7 +83,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -93,7 +93,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28789.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28788.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 2730 Data size: 646063 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) @@ -419,7 +419,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -428,7 +428,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28801.388)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28801.336)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 2730 Data size: 646063 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) diff --git ql/src/test/results/clientpositive/vectorization_6.q.out ql/src/test/results/clientpositive/vectorization_6.q.out index 8dedb63e7da..c33c9bb9f05 100644 --- ql/src/test/results/clientpositive/vectorization_6.q.out +++ ql/src/test/results/clientpositive/vectorization_6.q.out @@ -58,7 +58,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257))))) (type: boolean) + filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0))))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -68,7 +68,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprAndExpr(children: FilterLongColNotEqualLongScalar(col 0:tinyint, val 0), FilterExprOrExpr(children: FilterExprAndExpr(children: FilterLongColLessEqualLongScalar(col 10:boolean, val 0), FilterLongColGreaterEqualLongColumn(col 11:boolean, col 10:boolean)), FilterExprAndExpr(children: SelectColumnIsNotNull(col 3:bigint), FilterExprOrExpr(children: FilterStringColLikeStringScalar(col 7:string, pattern %a), FilterDoubleColLessEqualDoubleScalar(col 4:float, val -257.0))))) - predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257)))) and (ctinyint <> 0Y)) (type: boolean) + predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0)))) and (ctinyint <> 0Y)) (type: boolean) Statistics: Num rows: 11605 Data size: 2746359 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), cfloat (type: float), cstring1 (type: string), (988888 * UDFToInteger(csmallint)) (type: int), (- csmallint) (type: smallint), (- cfloat) (type: float), (-26.28D / UDFToDouble(cfloat)) (type: double), (cfloat * 359.0) (type: float), (cint % UDFToInteger(ctinyint)) (type: int), (- cdouble) (type: double), (UDFToInteger(ctinyint) - -75) (type: int), (762 * (cint % UDFToInteger(ctinyint))) (type: int) diff --git ql/src/test/results/clientpositive/vectorization_8.q.out ql/src/test/results/clientpositive/vectorization_8.q.out index d81df76a2f6..498506cbb7f 100644 --- ql/src/test/results/clientpositive/vectorization_8.q.out +++ ql/src/test/results/clientpositive/vectorization_8.q.out @@ -66,7 +66,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D)) or (cfloat < -6432) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) + filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D)) or (cfloat < -6432.0) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -76,7 +76,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 10.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 16.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0))) - predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D))) (type: boolean) + predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432.0) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) @@ -279,7 +279,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D)) or (cfloat < -6432) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) + filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D)) or (cfloat < -6432.0) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -288,7 +288,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 12.503)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 11.998)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0))) - predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D))) (type: boolean) + predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432.0) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double)