diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java index 29dc06dca1..f0b28c720d 100644 --- a/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java +++ b/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java @@ -43,6 +43,7 @@ public HiveChar(HiveChar hc, int len) { /** * Set char value, padding or truncating the value to the size of len parameter. */ + @Override public void setValue(String val, int len) { super.setValue(HiveBaseChar.getPaddedValue(val, len), -1); } @@ -59,15 +60,18 @@ public String getPaddedValue() { return value; } + @Override public int getCharacterLength() { String strippedValue = getStrippedValue(); return strippedValue.codePointCount(0, strippedValue.length()); } + @Override public String toString() { return getPaddedValue(); } + @Override public int compareTo(HiveChar rhs) { if (rhs == this) { return 0; @@ -75,6 +79,7 @@ public int compareTo(HiveChar rhs) { return this.getStrippedValue().compareTo(rhs.getStrippedValue()); } + @Override public boolean equals(Object rhs) { if (rhs == this) { return true; @@ -85,7 +90,9 @@ public boolean equals(Object rhs) { return this.getStrippedValue().equals(((HiveChar) rhs).getStrippedValue()); } + @Override public int hashCode() { return getStrippedValue().hashCode(); } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index e7d71595c7..1a58311530 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -18,13 +18,6 @@ package org.apache.hadoop.hive.ql; -import java.io.FileNotFoundException; -import java.text.MessageFormat; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - import org.antlr.runtime.tree.Tree; import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; @@ -35,6 +28,13 @@ import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.security.AccessControlException; +import java.io.FileNotFoundException; +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + /** * List of all error messages. * This list contains both compile time and run-time errors. @@ -469,6 +469,7 @@ LOAD_DATA_LAUNCH_JOB_PARSE_ERROR(10416, "Encountered parse error while parsing rewritten load data into insert query"), RESOURCE_PLAN_ALREADY_EXISTS(10417, "Resource plan {0} already exists", true), RESOURCE_PLAN_NOT_EXISTS(10418, "Resource plan {0} does not exist", true), + INCOMPATIBLE_STRUCT(10419, "Incompatible structs.", true), //========================== 20000 range starts here ========================// diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java index 04800cca91..d494e2342b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java @@ -17,14 +17,14 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.LinkedHashMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; +import com.google.common.collect.Sets; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; @@ -50,14 +50,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Function; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.LinkedHashMultimap; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Multimap; -import com.google.common.collect.Multimaps; -import com.google.common.collect.Sets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; public abstract class HivePointLookupOptimizerRule extends RelOptRule { @@ -223,6 +223,10 @@ public static Constraint of(RexNode n) { } RexNode opA = call.operands.get(0); RexNode opB = call.operands.get(1); + if (RexUtil.isNull(opA) || RexUtil.isNull(opB)) { + // dont try to compare nulls + return null; + } if (opA instanceof RexLiteral && opB instanceof RexInputRef) { RexLiteral rexLiteral = (RexLiteral) opA; RexInputRef rexInputRef = (RexInputRef) opB; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java index 1b56ecd044..12e7ae62f4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java @@ -19,9 +19,10 @@ package org.apache.hadoop.hive.ql.parse; import org.apache.calcite.rel.RelNode; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; + import java.util.Map; /** @@ -263,4 +264,8 @@ public boolean isUseCaching() { public boolean isFoldExpr() { return foldExpr; } + + public boolean isCBOExecuted() { + return foldExpr; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index 4968d16876..712d447e2d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -18,15 +18,7 @@ package org.apache.hadoop.hive.ql.parse; -import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Stack; +import com.google.common.collect.Lists; import org.apache.calcite.rel.RelNode; import org.apache.commons.lang.StringUtils; @@ -75,6 +67,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFNvl; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualNS; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen; @@ -105,7 +98,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.Lists; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Stack; /** @@ -353,14 +354,18 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } public static ExprNodeConstantDesc createDecimal(String strVal, boolean notNull) { - // Note: the normalize() call with rounding in HiveDecimal will currently reduce the - // precision and scale of the value by throwing away trailing zeroes. This may or may - // not be desirable for the literals; however, this used to be the default behavior - // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now. HiveDecimal hd = HiveDecimal.create(strVal); if (notNull && hd == null) { return null; } + return new ExprNodeConstantDesc(adjustType(hd), hd); + } + + private static DecimalTypeInfo adjustType(HiveDecimal hd) { + // Note: the normalize() call with rounding in HiveDecimal will currently reduce the + // precision and scale of the value by throwing away trailing zeroes. This may or may + // not be desirable for the literals; however, this used to be the default behavior + // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now. int prec = 1; int scale = 0; if (hd != null) { @@ -368,7 +373,7 @@ public static ExprNodeConstantDesc createDecimal(String strVal, boolean notNull) scale = hd.scale(); } DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale); - return new ExprNodeConstantDesc(typeInfo, hd); + return typeInfo; } } @@ -1163,36 +1168,40 @@ protected ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, ExprNodeDesc constChild = children.get(constIdx); ExprNodeDesc columnChild = children.get(1 - constIdx); - final PrimitiveTypeInfo colTypeInfo = - TypeInfoFactory.getPrimitiveTypeInfo(columnChild.getTypeString().toLowerCase()); - ExprNodeDesc newChild = interpretNodeAs(colTypeInfo, constChild); - if (newChild == null) { - // non-interpretabe as that type... - if (genericUDF instanceof GenericUDFOPEqual) { - return new ExprNodeConstantDesc(false); - } - } else { - children.set(constIdx, newChild); + final PrimitiveTypeInfo colTypeInfo = + TypeInfoFactory.getPrimitiveTypeInfo(columnChild.getTypeString().toLowerCase()); + ExprNodeDesc newChild = interpretNodeAs(colTypeInfo, constChild); + if (newChild == null) { + // non-interpretable as target type... + // TODO: all comparisons with null should result in null + if (genericUDF instanceof GenericUDFOPEqual + && !(genericUDF instanceof GenericUDFOPEqualNS)) { + return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, null); } + } else { + children.set(constIdx, newChild); + } } - if (genericUDF instanceof GenericUDFIn && children.get(0) instanceof ExprNodeColumnDesc) { - ExprNodeColumnDesc columnDesc = (ExprNodeColumnDesc) children.get(0); - final PrimitiveTypeInfo colTypeInfo = - TypeInfoFactory.getPrimitiveTypeInfo(columnDesc.getTypeString().toLowerCase()); - List outputOpList = children.subList(1, children.size()); - ArrayList inOperands = new ArrayList<>(outputOpList); - outputOpList.clear(); - - for (ExprNodeDesc oldChild : inOperands) { - if(oldChild !=null && oldChild instanceof ExprNodeConstantDesc) { - ExprNodeDesc newChild = interpretNodeAs(colTypeInfo, oldChild); - if(newChild == null) { - // non interpretable as target type; skip - continue; + if (genericUDF instanceof GenericUDFIn) { + if (ctx.isCBOExecuted()) { + ExprNodeDesc columnDesc = children.get(0); + for (int i = 1; i < children.size(); i++) { + ExprNodeDesc newChild = interpretNodeAsStruct(columnDesc, children.get(i)); + if (newChild == null) { + throw new SemanticException("Found null value in IN clause, it should not happen"); + } + children.set(i, newChild); + } + } else { + List orOperands = rewriteInToOR(children); + if (orOperands != null) { + if (orOperands.size() == 1) { + orOperands.add(new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, false)); } - outputOpList.add(newChild); - }else{ - outputOpList.add(oldChild); + funcText = "or"; + genericUDF = new GenericUDFOPOr(); + children.clear(); + children.addAll(orOperands); } } } @@ -1258,48 +1267,221 @@ protected ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, return desc; } - private ExprNodeDesc interpretNodeAs(PrimitiveTypeInfo colTypeInfo, ExprNodeDesc constChild) { + private ArrayList rewriteInToOR(ArrayList inOperands) + throws SemanticException { + ExprNodeDesc columnDesc = inOperands.get(0); + + ArrayList orOperands = new ArrayList<>(); + for (int i = 1; i < inOperands.size(); i++) { + ExprNodeDesc andExpr = buildEqualsArr(columnDesc, inOperands.get(i)); + if (andExpr == null) { + return null; + } + orOperands.add(andExpr); + } + return orOperands; + } + + private ExprNodeDesc buildEqualsArr(ExprNodeDesc columnDesc, ExprNodeDesc exprNodeDesc) throws SemanticException { + List lNodes = asListOfNodes(columnDesc); + List rNodes = asListOfNodes(exprNodeDesc); + if (lNodes == null || rNodes == null) { + // something went wrong + return null; + } + if(lNodes.size()!=rNodes.size()) { + throw new SemanticException(ErrorMsg.INCOMPATIBLE_STRUCT.getMsg(columnDesc + " and " + exprNodeDesc)); + } + + List ret = new ArrayList<>(); + for (int i = 0; i < lNodes.size(); i++) { + ret.add(buildEquals(lNodes.get(i), rNodes.get(i))); + } + return buildAnd(ret); + } + + private ExprNodeGenericFuncDesc buildEquals(ExprNodeDesc columnDesc, ExprNodeDesc valueDesc) { + final PrimitiveTypeInfo colTypeInfo = + TypeInfoFactory.getPrimitiveTypeInfo(columnDesc.getTypeString().toLowerCase()); + return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPEqual(), + Lists.newArrayList(columnDesc, interpretNodeAs(colTypeInfo, valueDesc))); + } + + private ExprNodeDesc buildAnd(List values) { + if (values.size() == 1) { + return values.get(0); + } else { + return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPAnd(), + values); + } + } + + private ExprNodeGenericFuncDesc buildOr(List values) { + return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPOr(), + values); + } + + private List asListOfNodes(ExprNodeDesc desc) { + if (desc instanceof ExprNodeColumnDesc) { + return Lists.newArrayList(desc); + } + ExprNodeDesc valueDesc = desc; + if (ExprNodeDescUtils.isStructUDF(desc)) { + List valueChilds = ((ExprNodeGenericFuncDesc) valueDesc).getChildren(); + return valueChilds; + } + if (ExprNodeDescUtils.isConstantStruct(valueDesc)) { + ExprNodeConstantDesc valueConstDesc = (ExprNodeConstantDesc) valueDesc; + List oldValues = (List) valueConstDesc.getValue(); + StructTypeInfo structTypeInfo = (StructTypeInfo) valueConstDesc.getTypeInfo(); + ArrayList structFieldInfos = structTypeInfo.getAllStructFieldTypeInfos(); + + ArrayList ret = new ArrayList<>(); + for (int i = 0; i < oldValues.size(); i++) { + ret.add(new ExprNodeConstantDesc(structFieldInfos.get(i), oldValues.get(i))); + } + return ret; + } + if (valueDesc instanceof ExprNodeConstantDesc) { + return Lists.newArrayList(valueDesc); + } + + return null; + } + + /** + * Interprets the given value as columnDesc if possible + */ + private static ExprNodeDesc interpretNodeAsStruct(ExprNodeDesc columnDesc, ExprNodeDesc valueDesc) + throws SemanticException { + if(columnDesc instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc exprNodeColumnDesc = (ExprNodeColumnDesc) columnDesc; + final PrimitiveTypeInfo typeInfo = + TypeInfoFactory.getPrimitiveTypeInfo(exprNodeColumnDesc.getTypeString().toLowerCase()); + return interpretNodeAs(typeInfo, valueDesc); + } + if (ExprNodeDescUtils.isStructUDF(columnDesc) && ExprNodeDescUtils.isConstantStruct(valueDesc)) { + List columnChilds = ((ExprNodeGenericFuncDesc) columnDesc).getChildren(); + ExprNodeConstantDesc valueConstDesc = (ExprNodeConstantDesc) valueDesc; + StructTypeInfo structTypeInfo = (StructTypeInfo) valueConstDesc.getTypeInfo(); + ArrayList structFieldInfos = structTypeInfo.getAllStructFieldTypeInfos(); + ArrayList newStructFieldInfos = new ArrayList<>(); + + if (columnChilds.size() != structFieldInfos.size()) { + throw new SemanticException(ErrorMsg.INCOMPATIBLE_STRUCT.getMsg(columnChilds + " and " + structFieldInfos)); + } + List oldValues = (List) valueConstDesc.getValue(); + List newValues = new ArrayList<>(); + for (int i = 0; i < columnChilds.size(); i++) { + newStructFieldInfos.add(columnChilds.get(i).getTypeInfo()); + Object newValue = interpretConstantAsPrimitive( + (PrimitiveTypeInfo) columnChilds.get(i).getTypeInfo(), + oldValues.get(i), + structFieldInfos.get(i)); + newValues.add(newValue); + } + StructTypeInfo sti = new StructTypeInfo(); + sti.setAllStructFieldTypeInfos(newStructFieldInfos); + sti.setAllStructFieldNames(structTypeInfo.getAllStructFieldNames()); + return new ExprNodeConstantDesc(sti, newValues); + + } + if (ExprNodeDescUtils.isStructUDF(columnDesc) && ExprNodeDescUtils.isStructUDF(valueDesc)) { + List columnChilds = ((ExprNodeGenericFuncDesc) columnDesc).getChildren(); + List valueChilds = ((ExprNodeGenericFuncDesc) valueDesc).getChildren(); + if (columnChilds.size() != valueChilds.size()) { + throw new SemanticException(ErrorMsg.INCOMPATIBLE_STRUCT.getMsg(columnChilds + " and " + valueChilds)); + } + List oldValueChilds = new ArrayList<>(valueChilds); + valueChilds.clear(); + for (int i = 0; i < oldValueChilds.size(); i++) { + ExprNodeDesc newValue = interpretNodeAsStruct(columnChilds.get(i), oldValueChilds.get(i)); + valueChilds.add(newValue); + } + } + return valueDesc; + } + + private static ExprNodeDesc interpretNodeAs(PrimitiveTypeInfo colTypeInfo, ExprNodeDesc constChild) { if (constChild instanceof ExprNodeConstantDesc) { // Try to narrow type of constant Object constVal = ((ExprNodeConstantDesc) constChild).getValue(); - String constType = constChild.getTypeString().toLowerCase(); - if (constVal instanceof Number || constVal instanceof String) { - try { - PrimitiveTypeEntry primitiveTypeEntry = colTypeInfo.getPrimitiveTypeEntry(); - if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) { - return new ExprNodeConstantDesc(new Integer(constVal.toString())); - } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) { - return new ExprNodeConstantDesc(new Long(constVal.toString())); - } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) { - return new ExprNodeConstantDesc(new Double(constVal.toString())); - } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) { - return new ExprNodeConstantDesc(new Float(constVal.toString())); - } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) { - return new ExprNodeConstantDesc(new Byte(constVal.toString())); - } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) { - return new ExprNodeConstantDesc(new Short(constVal.toString())); - } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) { - return NumExprProcessor.createDecimal(constVal.toString(), false); - } - } catch (NumberFormatException nfe) { - LOG.trace("Failed to narrow type of constant", nfe); - if (!NumberUtils.isNumber(constVal.toString())) { - return null; - } + if (constVal == null) { + // adjust type of null + return new ExprNodeConstantDesc(colTypeInfo, null); + } + Object newConst = interpretConstantAsPrimitive(colTypeInfo, constVal, constChild.getTypeInfo()); + if (newConst == null) { + return null; + } + if(newConst == constVal) { + return constChild; + } else { + return new ExprNodeConstantDesc(adjustType(colTypeInfo, newConst), newConst); + } + } + return constChild; + } + + private static TypeInfo adjustType(PrimitiveTypeInfo colTypeInfo, Object newConst) { + if (newConst instanceof HiveDecimal) { + return NumExprProcessor.adjustType((HiveDecimal) newConst); + } + return colTypeInfo; + } + + private static Object interpretConstantAsPrimitive(PrimitiveTypeInfo colTypeInfo, Object constVal, + TypeInfo constTypeInfo) { + String constTypeInfoName = constTypeInfo.getTypeName(); + if (constVal instanceof Number || constVal instanceof String) { + try { + PrimitiveTypeEntry primitiveTypeEntry = colTypeInfo.getPrimitiveTypeEntry(); + if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) { + return (new Integer(constVal.toString())); + } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) { + return (new Long(constVal.toString())); + } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) { + return (new Double(constVal.toString())); + } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) { + return (new Float(constVal.toString())); + } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) { + return (new Byte(constVal.toString())); + } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) { + return (new Short(constVal.toString())); + } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) { + return HiveDecimal.create(constVal.toString()); + } + } catch (NumberFormatException nfe) { + LOG.trace("Failed to narrow type of constant", nfe); + if (!NumberUtils.isNumber(constVal.toString())) { + return null; } } + } + + // Comparision of decimal and float/double happens in float/double. + if (constVal instanceof HiveDecimal) { + HiveDecimal hiveDecimal = (HiveDecimal) constVal; - // if column type is char and constant type is string, then convert the constant to char - // type with padded spaces. - if (constType.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME) && colTypeInfo instanceof CharTypeInfo) { - final Object originalValue = ((ExprNodeConstantDesc) constChild).getValue(); - final String constValue = originalValue.toString(); - final int length = TypeInfoUtils.getCharacterLengthForType(colTypeInfo); - final HiveChar newValue = new HiveChar(constValue, length); - return new ExprNodeConstantDesc(colTypeInfo, newValue); + PrimitiveTypeEntry primitiveTypeEntry = colTypeInfo.getPrimitiveTypeEntry(); + if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) { + return hiveDecimal.doubleValue(); + } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) { + return hiveDecimal.floatValue(); } + return hiveDecimal; } - return constChild; + + // TODO : Char and string comparison happens in char. But, varchar and string comparison happens in String. + + // if column type is char and constant type is string, then convert the constant to char + // type with padded spaces. + if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME) && colTypeInfo instanceof CharTypeInfo) { + final String constValue = constVal.toString(); + final int length = TypeInfoUtils.getCharacterLengthForType(colTypeInfo); + return new HiveChar(constValue, length); + } + return constVal; } private boolean canConvertIntoNvl(GenericUDF genericUDF, ArrayList children) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java index c274fd7cc9..596640ee49 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java @@ -18,18 +18,14 @@ package org.apache.hadoop.hive.ql.plan; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.ql.exec.RowSchema; +import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; @@ -38,6 +34,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -45,10 +42,15 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.util.ReflectionUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + public class ExprNodeDescUtils { @@ -571,7 +573,7 @@ public static void getExprNodeColumnDesc(List exprDescList, /** * Get Map of ExprNodeColumnDesc HashCode to ExprNodeColumnDesc. - * + * * @param exprDesc * @param hashCodeToColumnDescMap * Assumption: If two ExprNodeColumnDesc have same hash code then @@ -642,7 +644,9 @@ public static PrimitiveTypeInfo deriveMinArgumentCast( // We only do the minimum cast for decimals. Other types are assumed safe; fix if needed. // We also don't do anything for non-primitive children (maybe we should assert). if ((pti.getPrimitiveCategory() != PrimitiveCategory.DECIMAL) - || (!(childExpr.getTypeInfo() instanceof PrimitiveTypeInfo))) return pti; + || (!(childExpr.getTypeInfo() instanceof PrimitiveTypeInfo))) { + return pti; + } PrimitiveTypeInfo childTi = (PrimitiveTypeInfo)childExpr.getTypeInfo(); // If the child is also decimal, no cast is needed (we hope - can target type be narrower?). return HiveDecimalUtils.getDecimalTypeForPrimitiveCategory(childTi); @@ -652,7 +656,7 @@ public static PrimitiveTypeInfo deriveMinArgumentCast( * Build ExprNodeColumnDesc for the projections in the input operator from * sartpos to endpos(both included). Operator must have an associated * colExprMap. - * + * * @param inputOp * Input Hive Operator * @param startPos @@ -684,7 +688,7 @@ public static PrimitiveTypeInfo deriveMinArgumentCast( } return exprColLst; - } + } public static List flattenExprList(List sourceList) { ArrayList result = new ArrayList(sourceList.size()); @@ -998,4 +1002,17 @@ public static boolean isIntegerType(ExprNodeDesc expr) { } return false; } + + public static boolean isConstantStruct(ExprNodeDesc valueDesc) { + return valueDesc instanceof ExprNodeConstantDesc && valueDesc.getTypeInfo() instanceof StructTypeInfo; + } + + public static boolean isStructUDF(ExprNodeDesc columnDesc) { + if (columnDesc instanceof ExprNodeGenericFuncDesc) { + ExprNodeGenericFuncDesc exprNodeGenericFuncDesc = (ExprNodeGenericFuncDesc) columnDesc; + return (exprNodeGenericFuncDesc.getGenericUDF() instanceof GenericUDFStruct); + } + return false; + } + } diff --git a/ql/src/test/queries/clientpositive/in_typecheck_char.q b/ql/src/test/queries/clientpositive/in_typecheck_char.q new file mode 100644 index 0000000000..3955c4be14 --- /dev/null +++ b/ql/src/test/queries/clientpositive/in_typecheck_char.q @@ -0,0 +1,24 @@ + +create table ax(s char(1),t char(10)); + +insert into ax values ('a','a'),('a','a '),('b','bb'); + +explain +select 'expected 2',count(*) from ax where s = 'a' and t = 'a'; +select 'expected 2',count(*) from ax where s = 'a' and t = 'a'; + +explain +select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb')); +select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb')); + +select 'expected 2',count(*) from ax where t = 'a '; +select 'expected 2',count(*) from ax where t = 'a '; +select 'expected 0',count(*) from ax where t = 'a d'; + + +select 'expected 2',count(*) from ax where (s,t) in (('a','a'),(null, 'bb')); + + +-- this is right now broken; HIVE-20779 should fix it +explain select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null; +select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null; diff --git a/ql/src/test/queries/clientpositive/in_typecheck_pointlook.q b/ql/src/test/queries/clientpositive/in_typecheck_pointlook.q new file mode 100644 index 0000000000..c788506f45 --- /dev/null +++ b/ql/src/test/queries/clientpositive/in_typecheck_pointlook.q @@ -0,0 +1,35 @@ + +create table customer_demographics (cd_marital_status char(1), cd_education_status char(20)); + +insert into customer_demographics values +('M','Unknown'), +('W','Advanced Degree'), +('W','Advanced Degree '), +('W',' Advanced Degree') +; + +set hive.optimize.point.lookup.min=32; + +explain +select count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')); + +select '3 is expected:',count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')); + +set hive.optimize.point.lookup.min=2; + +explain +select count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')); + +select '3 is expected:',count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')); diff --git a/ql/src/test/queries/clientpositive/in_typecheck_varchar.q b/ql/src/test/queries/clientpositive/in_typecheck_varchar.q new file mode 100644 index 0000000000..e8dbf46bec --- /dev/null +++ b/ql/src/test/queries/clientpositive/in_typecheck_varchar.q @@ -0,0 +1,16 @@ + +create table ax(s varchar(1),t varchar(10)); + +insert into ax values ('a','a'),('a','a '),('b','bb'); + +explain +select 'expected 1',count(*) from ax where s = 'a' and t = 'a'; +select 'expected 1',count(*) from ax where s = 'a' and t = 'a'; + +explain +select 'expected 2',count(*) from ax where (s,t) in (('a','a'),('b','bb')); +select 'expected 2',count(*) from ax where (s,t) in (('a','a'),('b','bb')); + +select 'expected 0',count(*) from ax where t = 'a '; +select 'expected 0',count(*) from ax where t = 'a '; +select 'expected 0',count(*) from ax where t = 'a d'; diff --git a/ql/src/test/queries/clientpositive/in_typecheck_x1.q b/ql/src/test/queries/clientpositive/in_typecheck_x1.q new file mode 100644 index 0000000000..26c486bf52 --- /dev/null +++ b/ql/src/test/queries/clientpositive/in_typecheck_x1.q @@ -0,0 +1,12 @@ +create table t (a string); + +insert into t values ('1'),('x'),('2.0'); + +explain +select * from t where a in (1.0,'x',2); +select * from t where a in (1.0,'x',2); + + +create table ax(s char(1),t char(10)); +insert into ax values ('a','a'),('a','a '),('b','bb'); +select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null; diff --git a/ql/src/test/queries/clientpositive/join45X.q b/ql/src/test/queries/clientpositive/join45X.q new file mode 100644 index 0000000000..8291c9e3d8 --- /dev/null +++ b/ql/src/test/queries/clientpositive/join45X.q @@ -0,0 +1,32 @@ +--! qt:dataset:src +--! qt:dataset:src1 + +set hive.strict.checks.cartesian.product=false; + +-- Function with multiple inputs on one side +EXPLAIN +SELECT * +FROM src1 JOIN src +ON (src1.key= 100 and src.key=100) +LIMIT 10; + + +-- Function with multiple inputs on one side +EXPLAIN +SELECT * +FROM src1 JOIN src +ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) +LIMIT 10; + +SELECT * +FROM src1 JOIN src +ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) +LIMIT 10; + + +EXPLAIN + SELECT * + FROM src1 JOIN src + ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) + LIMIT 10 + diff --git a/ql/src/test/results/clientpositive/alter_partition_coltype.q.out b/ql/src/test/results/clientpositive/alter_partition_coltype.q.out index f6c3c5642e..d484f9e223 100644 --- a/ql/src/test/results/clientpositive/alter_partition_coltype.q.out +++ b/ql/src/test/results/clientpositive/alter_partition_coltype.q.out @@ -160,7 +160,7 @@ POSTHOOK: Input: default@alter_coltype #### A masked pattern was here #### OPTIMIZED SQL: SELECT COUNT(*) AS `$f0` FROM `default`.`alter_coltype` -WHERE `ts` = 3 AND `dt` = 100 +WHERE `ts` = 3.0 AND `dt` = 100 STAGE DEPENDENCIES: Stage-0 is a root stage diff --git a/ql/src/test/results/clientpositive/in_typecheck_char.q.out b/ql/src/test/results/clientpositive/in_typecheck_char.q.out new file mode 100644 index 0000000000..c605ffd0a8 --- /dev/null +++ b/ql/src/test/results/clientpositive/in_typecheck_char.q.out @@ -0,0 +1,261 @@ +PREHOOK: query: create table ax(s char(1),t char(10)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ax +POSTHOOK: query: create table ax(s char(1),t char(10)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ax +PREHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ax +POSTHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ax +POSTHOOK: Lineage: ax.s SCRIPT [] +POSTHOOK: Lineage: ax.t SCRIPT [] +PREHOOK: query: explain +select 'expected 2',count(*) from ax where s = 'a' and t = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: explain +select 'expected 2',count(*) from ax where s = 'a' and t = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: ax + filterExpr: ((s = 'a') and (t = 'a ')) (type: boolean) + Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((s = 'a') and (t = 'a ')) (type: boolean) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 'expected 2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select 'expected 2',count(*) from ax where s = 'a' and t = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 2',count(*) from ax where s = 'a' and t = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 2 2 +PREHOOK: query: explain +select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb')) +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: explain +select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: ax + filterExpr: (struct(s,t)) IN (const struct('a','a '), const struct('b','bb ')) (type: boolean) + Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (struct(s,t)) IN (const struct('a','a '), const struct('b','bb ')) (type: boolean) + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 'expected 3' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb')) +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 3 3 +PREHOOK: query: select 'expected 2',count(*) from ax where t = 'a ' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 2',count(*) from ax where t = 'a ' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 2 2 +PREHOOK: query: select 'expected 2',count(*) from ax where t = 'a ' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 2',count(*) from ax where t = 'a ' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 2 2 +PREHOOK: query: select 'expected 0',count(*) from ax where t = 'a d' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 0',count(*) from ax where t = 'a d' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 0 2 +PREHOOK: query: select 'expected 2',count(*) from ax where (s,t) in (('a','a'),(null, 'bb')) +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 2',count(*) from ax where (s,t) in (('a','a'),(null, 'bb')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 2 2 +PREHOOK: query: explain select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: explain select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: ax + filterExpr: (((s = 'a') and (t = 'a ')) or ((s = null) and (t = 'bb '))) is null (type: boolean) + Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((s = 'a') and (t = 'a ')) or ((s = null) and (t = 'bb '))) is null (type: boolean) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 'expected 1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 1 1 diff --git a/ql/src/test/results/clientpositive/in_typecheck_pointlook.q.out b/ql/src/test/results/clientpositive/in_typecheck_pointlook.q.out new file mode 100644 index 0000000000..7a913fec97 --- /dev/null +++ b/ql/src/test/results/clientpositive/in_typecheck_pointlook.q.out @@ -0,0 +1,181 @@ +PREHOOK: query: create table customer_demographics (cd_marital_status char(1), cd_education_status char(20)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@customer_demographics +POSTHOOK: query: create table customer_demographics (cd_marital_status char(1), cd_education_status char(20)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@customer_demographics +PREHOOK: query: insert into customer_demographics values +('M','Unknown'), +('W','Advanced Degree'), +('W','Advanced Degree '), +('W',' Advanced Degree') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@customer_demographics +POSTHOOK: query: insert into customer_demographics values +('M','Unknown'), +('W','Advanced Degree'), +('W','Advanced Degree '), +('W',' Advanced Degree') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@customer_demographics +POSTHOOK: Lineage: customer_demographics.cd_education_status SCRIPT [] +POSTHOOK: Lineage: customer_demographics.cd_marital_status SCRIPT [] +PREHOOK: query: explain +select count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +PREHOOK: type: QUERY +PREHOOK: Input: default@customer_demographics +#### A masked pattern was here #### +POSTHOOK: query: explain +select count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@customer_demographics +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: customer_demographics + filterExpr: (((cd_marital_status = 'M') and (cd_education_status = 'Unknown ')) or ((cd_marital_status = 'W') and (cd_education_status = 'Advanced Degree '))) (type: boolean) + Statistics: Num rows: 4 Data size: 88 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((cd_marital_status = 'M') and (cd_education_status = 'Unknown ')) or ((cd_marital_status = 'W') and (cd_education_status = 'Advanced Degree '))) (type: boolean) + Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select '3 is expected:',count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +PREHOOK: type: QUERY +PREHOOK: Input: default@customer_demographics +#### A masked pattern was here #### +POSTHOOK: query: select '3 is expected:',count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@customer_demographics +#### A masked pattern was here #### +3 is expected: 3 +PREHOOK: query: explain +select count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +PREHOOK: type: QUERY +PREHOOK: Input: default@customer_demographics +#### A masked pattern was here #### +POSTHOOK: query: explain +select count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@customer_demographics +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: customer_demographics + filterExpr: (struct(cd_marital_status,cd_education_status)) IN (const struct('M','Unknown '), const struct('W','Advanced Degree ')) (type: boolean) + Statistics: Num rows: 4 Data size: 88 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (struct(cd_marital_status,cd_education_status)) IN (const struct('M','Unknown '), const struct('W','Advanced Degree ')) (type: boolean) + Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select '3 is expected:',count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +PREHOOK: type: QUERY +PREHOOK: Input: default@customer_demographics +#### A masked pattern was here #### +POSTHOOK: query: select '3 is expected:',count(1) +from customer_demographics +where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') +or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@customer_demographics +#### A masked pattern was here #### +3 is expected: 3 diff --git a/ql/src/test/results/clientpositive/in_typecheck_varchar.q.out b/ql/src/test/results/clientpositive/in_typecheck_varchar.q.out new file mode 100644 index 0000000000..b208fbf05b --- /dev/null +++ b/ql/src/test/results/clientpositive/in_typecheck_varchar.q.out @@ -0,0 +1,185 @@ +PREHOOK: query: create table ax(s varchar(1),t varchar(10)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ax +POSTHOOK: query: create table ax(s varchar(1),t varchar(10)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ax +PREHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ax +POSTHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ax +POSTHOOK: Lineage: ax.s SCRIPT [] +POSTHOOK: Lineage: ax.t SCRIPT [] +PREHOOK: query: explain +select 'expected 1',count(*) from ax where s = 'a' and t = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: explain +select 'expected 1',count(*) from ax where s = 'a' and t = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: ax + filterExpr: ((CAST( s AS STRING) = 'a') and (CAST( t AS STRING) = 'a')) (type: boolean) + Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((CAST( s AS STRING) = 'a') and (CAST( t AS STRING) = 'a')) (type: boolean) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 'expected 1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select 'expected 1',count(*) from ax where s = 'a' and t = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 1',count(*) from ax where s = 'a' and t = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 1 1 +PREHOOK: query: explain +select 'expected 2',count(*) from ax where (s,t) in (('a','a'),('b','bb')) +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: explain +select 'expected 2',count(*) from ax where (s,t) in (('a','a'),('b','bb')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: ax + filterExpr: (((CAST( s AS STRING) = 'a') and (CAST( t AS STRING) = 'a')) or ((CAST( s AS STRING) = 'b') and (CAST( t AS STRING) = 'bb'))) (type: boolean) + Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((CAST( s AS STRING) = 'a') and (CAST( t AS STRING) = 'a')) or ((CAST( s AS STRING) = 'b') and (CAST( t AS STRING) = 'bb'))) (type: boolean) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 'expected 2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select 'expected 2',count(*) from ax where (s,t) in (('a','a'),('b','bb')) +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 2',count(*) from ax where (s,t) in (('a','a'),('b','bb')) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 2 2 +PREHOOK: query: select 'expected 0',count(*) from ax where t = 'a ' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 0',count(*) from ax where t = 'a ' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 0 0 +PREHOOK: query: select 'expected 0',count(*) from ax where t = 'a ' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 0',count(*) from ax where t = 'a ' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 0 0 +PREHOOK: query: select 'expected 0',count(*) from ax where t = 'a d' +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 0',count(*) from ax where t = 'a d' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 0 0 diff --git a/ql/src/test/results/clientpositive/in_typecheck_x1.q.out b/ql/src/test/results/clientpositive/in_typecheck_x1.q.out new file mode 100644 index 0000000000..4eebc1218e --- /dev/null +++ b/ql/src/test/results/clientpositive/in_typecheck_x1.q.out @@ -0,0 +1,99 @@ +PREHOOK: query: create table t (a string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: create table t (a string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +PREHOOK: query: insert into t values ('1'),('x'),('2.0') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@t +POSTHOOK: query: insert into t values ('1'),('x'),('2.0') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@t +POSTHOOK: Lineage: t.a SCRIPT [] +PREHOOK: query: explain +select * from t where a in (1.0,'x',2) +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: explain +select * from t where a in (1.0,'x',2) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: t + filterExpr: ((UDFToDouble(a) = 1.0D) or (a = 'x') or (UDFToDouble(a) = 2.0D)) (type: boolean) + Statistics: Num rows: 3 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(a) = 1.0D) or (UDFToDouble(a) = 2.0D) or (a = 'x')) (type: boolean) + Statistics: Num rows: 3 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 5 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 5 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from t where a in (1.0,'x',2) +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select * from t where a in (1.0,'x',2) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +1 +x +2.0 +PREHOOK: query: create table ax(s char(1),t char(10)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ax +POSTHOOK: query: create table ax(s char(1),t char(10)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ax +PREHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ax +POSTHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ax +POSTHOOK: Lineage: ax.s SCRIPT [] +POSTHOOK: Lineage: ax.t SCRIPT [] +PREHOOK: query: select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null +PREHOOK: type: QUERY +PREHOOK: Input: default@ax +#### A masked pattern was here #### +POSTHOOK: query: select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ax +#### A masked pattern was here #### +expected 1 1 diff --git a/ql/src/test/results/clientpositive/infer_const_type.q.out b/ql/src/test/results/clientpositive/infer_const_type.q.out index e1d7de5422..db3b012951 100644 --- a/ql/src/test/results/clientpositive/infer_const_type.q.out +++ b/ql/src/test/results/clientpositive/infer_const_type.q.out @@ -61,10 +61,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: infertypes - filterExpr: ((ti = 127Y) and (si = 32767S) and (i = 12345) and (bi = -12345L) and (fl = 906) and (db = -307.0D) and (UDFToDouble(str) = 1234.0D)) (type: boolean) + filterExpr: ((ti = 127Y) and (si = 32767S) and (i = 12345) and (bi = -12345L) and (fl = 906.0) and (db = -307.0D) and (UDFToDouble(str) = 1234.0D)) (type: boolean) Statistics: Num rows: 1 Data size: 1170 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((UDFToDouble(str) = 1234.0D) and (bi = -12345L) and (db = -307.0D) and (fl = 906) and (i = 12345) and (si = 32767S) and (ti = 127Y)) (type: boolean) + predicate: ((UDFToDouble(str) = 1234.0D) and (bi = -12345L) and (db = -307.0D) and (fl = 906.0) and (i = 12345) and (si = 32767S) and (ti = 127Y)) (type: boolean) Statistics: Num rows: 1 Data size: 1170 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 127Y (type: tinyint), 32767S (type: smallint), 12345 (type: int), -12345L (type: bigint), 906.0 (type: float), -307.0D (type: double), str (type: string) @@ -139,10 +139,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: infertypes - filterExpr: ((UDFToDouble(ti) = 128.0D) or (UDFToInteger(si) = 32768) or (UDFToDouble(i) = 2.147483648E9D) or (UDFToDouble(bi) = 9.223372036854776E18D)) (type: boolean) + filterExpr: ((UDFToDouble(ti) = 128.0D) or (UDFToInteger(si) = 32768) or (UDFToDouble(i) = 2.147483648E9D) or (UDFToDouble(bi) = 9.223372036854776E18D) or null) (type: boolean) Statistics: Num rows: 1 Data size: 1170 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((UDFToDouble(bi) = 9.223372036854776E18D) or (UDFToDouble(i) = 2.147483648E9D) or (UDFToDouble(ti) = 128.0D) or (UDFToInteger(si) = 32768)) (type: boolean) + predicate: ((UDFToDouble(bi) = 9.223372036854776E18D) or (UDFToDouble(i) = 2.147483648E9D) or (UDFToDouble(ti) = 128.0D) or (UDFToInteger(si) = 32768) or null) (type: boolean) Statistics: Num rows: 1 Data size: 1170 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ti (type: tinyint), si (type: smallint), i (type: int), bi (type: bigint), fl (type: float), db (type: double), str (type: string) diff --git a/ql/src/test/results/clientpositive/join45.q.out b/ql/src/test/results/clientpositive/join45.q.out index 47aaf7d0ab..d229bbba55 100644 --- a/ql/src/test/results/clientpositive/join45.q.out +++ b/ql/src/test/results/clientpositive/join45.q.out @@ -714,7 +714,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 12500 Data size: 240800 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (struct(_col0,_col2)) IN (const struct(100,100), const struct(101,101), const struct(102,102)) (type: boolean) + predicate: (((UDFToDouble(_col0) = 100.0D) and (UDFToDouble(_col2) = 100.0D)) or ((UDFToDouble(_col0) = 101.0D) and (UDFToDouble(_col2) = 101.0D)) or ((UDFToDouble(_col0) = 102.0D) and (UDFToDouble(_col2) = 102.0D))) (type: boolean) Statistics: Num rows: 9375 Data size: 180600 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 10 diff --git a/ql/src/test/results/clientpositive/join45X.q.out b/ql/src/test/results/clientpositive/join45X.q.out new file mode 100644 index 0000000000..880a2e3a3f --- /dev/null +++ b/ql/src/test/results/clientpositive/join45X.q.out @@ -0,0 +1,256 @@ +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +PREHOOK: query: EXPLAIN +SELECT * +FROM src1 JOIN src +ON (src1.key= 100 and src.key=100) +LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * +FROM src1 JOIN src +ON (src1.key= 100 and src.key=100) +LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src1 + filterExpr: (UDFToDouble(key) = 100.0D) (type: boolean) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) = 100.0D) (type: boolean) + Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + TableScan + alias: src + filterExpr: (UDFToDouble(key) = 100.0D) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) = 100.0D) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 3000 Data size: 57622 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 190 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 190 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + ListSink + +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +PREHOOK: query: EXPLAIN +SELECT * +FROM src1 JOIN src +ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) +LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * +FROM src1 JOIN src +ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) +LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 12500 Data size: 240800 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToDouble(_col0) = 100.0D) and (UDFToDouble(_col2) = 100.0D)) or ((UDFToDouble(_col0) = 101.0D) and (UDFToDouble(_col2) = 101.0D)) or ((UDFToDouble(_col0) = 102.0D) and (UDFToDouble(_col2) = 102.0D))) (type: boolean) + Statistics: Num rows: 9375 Data size: 180600 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 190 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 190 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + ListSink + +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +PREHOOK: query: SELECT * +FROM src1 JOIN src +ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) +LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM src1 JOIN src +ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) +LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +PREHOOK: query: EXPLAIN + SELECT * + FROM src1 JOIN src + ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) + LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN + SELECT * + FROM src1 JOIN src + ON ((src1.key,src.key) IN ((100,100),(101,101),(102,102))) + LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 12500 Data size: 240800 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToDouble(_col0) = 100.0D) and (UDFToDouble(_col2) = 100.0D)) or ((UDFToDouble(_col0) = 101.0D) and (UDFToDouble(_col2) = 101.0D)) or ((UDFToDouble(_col0) = 102.0D) and (UDFToDouble(_col2) = 102.0D))) (type: boolean) + Statistics: Num rows: 9375 Data size: 180600 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 190 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 190 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + ListSink + diff --git a/ql/src/test/results/clientpositive/join47.q.out b/ql/src/test/results/clientpositive/join47.q.out index 4d9e937815..1930eb9fb4 100644 --- a/ql/src/test/results/clientpositive/join47.q.out +++ b/ql/src/test/results/clientpositive/join47.q.out @@ -706,7 +706,7 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3 - residual filter predicates: {(struct(_col0,_col2)) IN (const struct(100,100), const struct(101,101), const struct(102,102))} + residual filter predicates: {(((UDFToDouble(_col0) = 100.0D) and (UDFToDouble(_col2) = 100.0D)) or ((UDFToDouble(_col0) = 101.0D) and (UDFToDouble(_col2) = 101.0D)) or ((UDFToDouble(_col0) = 102.0D) and (UDFToDouble(_col2) = 102.0D)))} Statistics: Num rows: 9375 Data size: 180600 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 10 diff --git a/ql/src/test/results/clientpositive/llap/dec_str.q.out b/ql/src/test/results/clientpositive/llap/dec_str.q.out index 554031e952..ce509a7597 100644 --- a/ql/src/test/results/clientpositive/llap/dec_str.q.out +++ b/ql/src/test/results/clientpositive/llap/dec_str.q.out @@ -102,11 +102,10 @@ STAGE PLANS: Processor Tree: TableScan alias: t1 - filterExpr: (a = null) (type: boolean) Filter Operator - predicate: (a = null) (type: boolean) + predicate: false (type: boolean) Select Operator - expressions: null (type: decimal(3,1)) + expressions: a (type: decimal(3,1)) outputColumnNames: _col0 ListSink @@ -128,11 +127,10 @@ STAGE PLANS: Processor Tree: TableScan alias: t1 - filterExpr: (a = null) (type: boolean) Filter Operator - predicate: (a = null) (type: boolean) + predicate: false (type: boolean) Select Operator - expressions: null (type: decimal(3,1)) + expressions: a (type: decimal(3,1)) outputColumnNames: _col0 ListSink diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out index f240468558..71838eeda5 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -484,7 +484,7 @@ Stage-0 Group By Operator [GBY_6] (rows=2 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_37] (rows=5 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 8 [SIMPLE_EDGE] llap @@ -500,7 +500,7 @@ Stage-0 Group By Operator [GBY_13] (rows=2 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_38] (rows=5 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -574,7 +574,7 @@ Stage-0 Group By Operator [GBY_6] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_37] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 8 [SIMPLE_EDGE] llap @@ -590,7 +590,7 @@ Stage-0 Group By Operator [GBY_13] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_38] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -657,7 +657,7 @@ Stage-0 Group By Operator [GBY_6] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_34] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 7 [SIMPLE_EDGE] llap @@ -673,7 +673,7 @@ Stage-0 Group By Operator [GBY_13] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_35] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -745,7 +745,7 @@ Stage-0 Group By Operator [GBY_6] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_36] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 8 [SIMPLE_EDGE] llap @@ -761,7 +761,7 @@ Stage-0 Group By Operator [GBY_13] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_37] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -828,7 +828,7 @@ Stage-0 Group By Operator [GBY_6] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_34] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 7 [SIMPLE_EDGE] llap @@ -844,7 +844,7 @@ Stage-0 Group By Operator [GBY_13] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_35] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -1315,7 +1315,7 @@ Stage-0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1","_col2"] Filter Operator [FIL_21] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 3 [SIMPLE_EDGE] llap @@ -1324,7 +1324,7 @@ Stage-0 Select Operator [SEL_5] (rows=9 width=89) Output:["_col0","_col1"] Filter Operator [FIL_22] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 4 [SIMPLE_EDGE] llap @@ -1374,7 +1374,7 @@ Stage-0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1","_col2"] Filter Operator [FIL_21] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 3 [SIMPLE_EDGE] llap @@ -1383,7 +1383,7 @@ Stage-0 Select Operator [SEL_5] (rows=9 width=89) Output:["_col0","_col1"] Filter Operator [FIL_22] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 4 [SIMPLE_EDGE] llap @@ -1640,7 +1640,7 @@ Stage-0 Group By Operator [GBY_3] (rows=3 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_50] (rows=6 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0))) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) TableScan [TS_0] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 9 [SIMPLE_EDGE] llap @@ -1664,7 +1664,7 @@ Stage-0 Group By Operator [GBY_16] (rows=3 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_52] (rows=6 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0))) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) TableScan [TS_13] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -1703,7 +1703,7 @@ Stage-0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1"] Filter Operator [FIL_15] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 3 [SIMPLE_EDGE] llap @@ -1755,7 +1755,7 @@ Stage-0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1","_col2"] Filter Operator [FIL_25] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 3 [SIMPLE_EDGE] llap @@ -1766,7 +1766,7 @@ Stage-0 Select Operator [SEL_5] (rows=9 width=85) Output:["_col0"] Filter Operator [FIL_26] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 4 [SIMPLE_EDGE] llap @@ -1855,7 +1855,7 @@ Stage-0 Group By Operator [GBY_3] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_41] (rows=1 width=93) - predicate:((((c_int + 1) + 1) >= 0) and (((c_int + 1) > 0) or (UDFToDouble(key) >= 0.0D)) and ((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (UDFToDouble(key) > 0.0D) and (c_float > 0)) + predicate:((((c_int + 1) + 1) >= 0) and (((c_int + 1) > 0) or (UDFToDouble(key) >= 0.0D)) and ((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (UDFToDouble(key) > 0.0D) and (c_float > 0.0)) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 7 [SIMPLE_EDGE] llap @@ -1873,7 +1873,7 @@ Stage-0 Group By Operator [GBY_12] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_42] (rows=1 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (UDFToDouble(key) > 0.0D) and (c_float > 0)) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (UDFToDouble(key) > 0.0D) and (c_float > 0.0)) TableScan [TS_9] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] diff --git a/ql/src/test/results/clientpositive/llap/lineage3.q.out b/ql/src/test/results/clientpositive/llap/lineage3.q.out index 9bec309c9c..94c6a13c99 100644 --- a/ql/src/test/results/clientpositive/llap/lineage3.q.out +++ b/ql/src/test/results/clientpositive/llap/lineage3.q.out @@ -317,7 +317,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc PREHOOK: Input: default@dest_v3 #### A masked pattern was here #### -{"version":"1.0","engine":"tez","database":"default","hash":"fd4e0dd59f42b53fc07125817451df49","queryText":"select * from dest_v3 limit 2","edges":[{"sources":[3,4,5,6,7],"targets":[0],"expression":"(tok_function sum (. (tok_table_or_col $hdt$_0) ctinyint) (tok_windowspec (tok_partitioningspec (tok_distributeby (. (tok_table_or_col $hdt$_0) csmallint)) (tok_orderby (tok_tabsortcolnameasc (tok_nulls_last (. (tok_table_or_col $hdt$_0) csmallint))))) (tok_windowvalues (preceding 2147483647) current)))","edgeType":"PROJECTION"},{"sources":[6],"targets":[1],"expression":"count(default.alltypesorc.cstring1)","edgeType":"PROJECTION"},{"sources":[5],"targets":[2],"edgeType":"PROJECTION"},{"sources":[8,7],"targets":[0,1,2],"expression":"(a.cboolean2 and a.cint is not null)","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2],"expression":"(a.cint = b.cint)","edgeType":"PREDICATE"},{"sources":[9,7],"targets":[0,1,2],"expression":"((b.cfloat > 0) and b.cint is not null)","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2],"expression":"(count(default.alltypesorc.cint) > 10L)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"dest_v3.a"},{"id":1,"vertexType":"COLUMN","vertexId":"dest_v3.x"},{"id":2,"vertexType":"COLUMN","vertexId":"dest_v3.cboolean1"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.csmallint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":6,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":7,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":8,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean2"},{"id":9,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cfloat"}]} +{"version":"1.0","engine":"tez","database":"default","hash":"fd4e0dd59f42b53fc07125817451df49","queryText":"select * from dest_v3 limit 2","edges":[{"sources":[3,4,5,6,7],"targets":[0],"expression":"(tok_function sum (. (tok_table_or_col $hdt$_0) ctinyint) (tok_windowspec (tok_partitioningspec (tok_distributeby (. (tok_table_or_col $hdt$_0) csmallint)) (tok_orderby (tok_tabsortcolnameasc (tok_nulls_last (. (tok_table_or_col $hdt$_0) csmallint))))) (tok_windowvalues (preceding 2147483647) current)))","edgeType":"PROJECTION"},{"sources":[6],"targets":[1],"expression":"count(default.alltypesorc.cstring1)","edgeType":"PROJECTION"},{"sources":[5],"targets":[2],"edgeType":"PROJECTION"},{"sources":[8,7],"targets":[0,1,2],"expression":"(a.cboolean2 and a.cint is not null)","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2],"expression":"(a.cint = b.cint)","edgeType":"PREDICATE"},{"sources":[9,7],"targets":[0,1,2],"expression":"((b.cfloat > 0.0) and b.cint is not null)","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2],"expression":"(count(default.alltypesorc.cint) > 10L)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"dest_v3.a"},{"id":1,"vertexType":"COLUMN","vertexId":"dest_v3.x"},{"id":2,"vertexType":"COLUMN","vertexId":"dest_v3.cboolean1"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.csmallint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":6,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":7,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":8,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean2"},{"id":9,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cfloat"}]} 38 216 false 38 229 true PREHOOK: query: drop table if exists src_dp diff --git a/ql/src/test/results/clientpositive/llap/vectorization_13.q.out b/ql/src/test/results/clientpositive/llap/vectorization_13.q.out index 4ce654f960..d84d0b42e3 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_13.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_13.q.out @@ -89,7 +89,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 2028982 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -99,7 +99,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28789.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28788.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 5461 Data size: 901772 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) @@ -448,7 +448,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 2028982 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -457,7 +457,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28801.388)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28801.336)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 5461 Data size: 901772 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) diff --git a/ql/src/test/results/clientpositive/llap/vectorization_6.q.out b/ql/src/test/results/clientpositive/llap/vectorization_6.q.out index a2f730beca..2990bd4b82 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_6.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_6.q.out @@ -61,7 +61,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257))))) (type: boolean) + filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0))))) (type: boolean) Statistics: Num rows: 12288 Data size: 2110130 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -71,7 +71,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprAndExpr(children: FilterLongColNotEqualLongScalar(col 0:tinyint, val 0), FilterExprOrExpr(children: FilterExprAndExpr(children: FilterLongColLessEqualLongScalar(col 10:boolean, val 0), FilterLongColGreaterEqualLongColumn(col 11:boolean, col 10:boolean)), FilterExprAndExpr(children: SelectColumnIsNotNull(col 3:bigint), FilterExprOrExpr(children: FilterStringColLikeStringScalar(col 7:string, pattern %a), FilterDoubleColLessEqualDoubleScalar(col 4:float, val -257.0))))) - predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257)))) and (ctinyint <> 0Y)) (type: boolean) + predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0)))) and (ctinyint <> 0Y)) (type: boolean) Statistics: Num rows: 5951 Data size: 1022000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: cboolean1 (type: boolean), cfloat (type: float), cstring1 (type: string), (988888 * UDFToInteger(csmallint)) (type: int), (- csmallint) (type: smallint), (- cfloat) (type: float), (-26.28D / UDFToDouble(cfloat)) (type: double), (cfloat * 359.0) (type: float), (cint % UDFToInteger(ctinyint)) (type: int), (- cdouble) (type: double), (UDFToInteger(ctinyint) - -75) (type: int), (762 * (cint % UDFToInteger(ctinyint))) (type: int) diff --git a/ql/src/test/results/clientpositive/llap/vectorization_8.q.out b/ql/src/test/results/clientpositive/llap/vectorization_8.q.out index 21ce7b8ebd..a0e1a4b214 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_8.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_8.q.out @@ -72,7 +72,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D)) or (cfloat < -6432) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) + filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D)) or (cfloat < -6432.0) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 2983078 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -82,7 +82,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 10.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 16.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0))) - predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D))) (type: boolean) + predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432.0) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D))) (type: boolean) Statistics: Num rows: 3059 Data size: 742850 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) @@ -313,7 +313,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D)) or (cfloat < -6432) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) + filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D)) or (cfloat < -6432.0) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 2983078 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -322,7 +322,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 12.503)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 11.998)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0))) - predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D))) (type: boolean) + predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432.0) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D))) (type: boolean) Statistics: Num rows: 3059 Data size: 742850 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) diff --git a/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out index 7f1c6a295e..f929706757 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out @@ -622,7 +622,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((ctimestamp1 = ctimestamp2) or (cfloat = 762) or (cstring1 = 'ss') or ((UDFToLong(csmallint) <= cbigint) and (cboolean2 = 1)) or (cboolean1 is not null and ctimestamp2 is not null and (cstring2 > 'a'))) (type: boolean) + filterExpr: ((ctimestamp1 = ctimestamp2) or (cfloat = 762.0) or (cstring1 = 'ss') or ((UDFToLong(csmallint) <= cbigint) and (cboolean2 = 1)) or (cboolean1 is not null and ctimestamp2 is not null and (cstring2 > 'a'))) (type: boolean) Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -631,7 +631,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterTimestampColEqualTimestampColumn(col 8:timestamp, col 9:timestamp), FilterDoubleColEqualDoubleScalar(col 4:float, val 762.0), FilterStringGroupColEqualStringScalar(col 6:string, val ss), FilterExprAndExpr(children: FilterLongColLessEqualLongColumn(col 1:bigint, col 3:bigint)(children: col 1:smallint), FilterLongColEqualLongScalar(col 11:boolean, val 1)), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), SelectColumnIsNotNull(col 9:timestamp), FilterStringGroupColGreaterStringScalar(col 7:string, val a))) - predicate: (((UDFToLong(csmallint) <= cbigint) and (cboolean2 = 1)) or (cboolean1 is not null and ctimestamp2 is not null and (cstring2 > 'a')) or (cfloat = 762) or (cstring1 = 'ss') or (ctimestamp1 = ctimestamp2)) (type: boolean) + predicate: (((UDFToLong(csmallint) <= cbigint) and (cboolean2 = 1)) or (cboolean1 is not null and ctimestamp2 is not null and (cstring2 > 'a')) or (cfloat = 762.0) or (cstring1 = 'ss') or (ctimestamp1 = ctimestamp2)) (type: boolean) Statistics: Num rows: 11346 Data size: 2856120 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: cbigint (type: bigint), ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cdouble (type: double), UDFToDouble(cbigint) (type: double), (UDFToDouble(cbigint) * UDFToDouble(cbigint)) (type: double), UDFToDouble(csmallint) (type: double), (UDFToDouble(csmallint) * UDFToDouble(csmallint)) (type: double), (cdouble * cdouble) (type: double) @@ -860,7 +860,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: (((ctimestamp2 <= ctimestamp1) and (UDFToDouble(cbigint) <> cdouble) and (cstring1 >= 'ss')) or ((csmallint < UDFToShort(ctinyint)) and (UDFToDouble(ctimestamp1) >= 0.0D)) or (cfloat = 17)) (type: boolean) + filterExpr: (((ctimestamp2 <= ctimestamp1) and (UDFToDouble(cbigint) <> cdouble) and (cstring1 >= 'ss')) or ((csmallint < UDFToShort(ctinyint)) and (UDFToDouble(ctimestamp1) >= 0.0D)) or (cfloat = 17.0)) (type: boolean) Statistics: Num rows: 12288 Data size: 2139070 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -869,7 +869,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterTimestampColLessEqualTimestampColumn(col 9:timestamp, col 8:timestamp), FilterDoubleColNotEqualDoubleColumn(col 13:double, col 5:double)(children: CastLongToDouble(col 3:bigint) -> 13:double), FilterStringGroupColGreaterEqualStringScalar(col 6:string, val ss)), FilterExprAndExpr(children: FilterLongColLessLongColumn(col 1:smallint, col 0:smallint)(children: col 0:tinyint), FilterDoubleColGreaterEqualDoubleScalar(col 13:double, val 0.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double)), FilterDoubleColEqualDoubleScalar(col 4:float, val 17.0)) - predicate: (((csmallint < UDFToShort(ctinyint)) and (UDFToDouble(ctimestamp1) >= 0.0D)) or ((ctimestamp2 <= ctimestamp1) and (UDFToDouble(cbigint) <> cdouble) and (cstring1 >= 'ss')) or (cfloat = 17)) (type: boolean) + predicate: (((csmallint < UDFToShort(ctinyint)) and (UDFToDouble(ctimestamp1) >= 0.0D)) or ((ctimestamp2 <= ctimestamp1) and (UDFToDouble(cbigint) <> cdouble) and (cstring1 >= 'ss')) or (cfloat = 17.0)) (type: boolean) Statistics: Num rows: 2824 Data size: 491654 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ctinyint (type: tinyint), cbigint (type: bigint), cint (type: int), cfloat (type: float), UDFToDouble(cint) (type: double), (UDFToDouble(cint) * UDFToDouble(cint)) (type: double), UDFToDouble(cbigint) (type: double), (UDFToDouble(cbigint) * UDFToDouble(cbigint)) (type: double) diff --git a/ql/src/test/results/clientpositive/mapjoin47.q.out b/ql/src/test/results/clientpositive/mapjoin47.q.out index 294dd69de5..0b40f1137f 100644 --- a/ql/src/test/results/clientpositive/mapjoin47.q.out +++ b/ql/src/test/results/clientpositive/mapjoin47.q.out @@ -748,7 +748,7 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3 - residual filter predicates: {(struct(_col0,_col2)) IN (const struct(100,100), const struct(101,101), const struct(102,102))} + residual filter predicates: {(((UDFToDouble(_col0) = 100.0D) and (UDFToDouble(_col2) = 100.0D)) or ((UDFToDouble(_col0) = 101.0D) and (UDFToDouble(_col2) = 101.0D)) or ((UDFToDouble(_col0) = 102.0D) and (UDFToDouble(_col2) = 102.0D)))} Statistics: Num rows: 9375 Data size: 180600 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 10 diff --git a/ql/src/test/results/clientpositive/parquet_vectorization_13.q.out b/ql/src/test/results/clientpositive/parquet_vectorization_13.q.out index 0efce98b55..0bd81bb886 100644 --- a/ql/src/test/results/clientpositive/parquet_vectorization_13.q.out +++ b/ql/src/test/results/clientpositive/parquet_vectorization_13.q.out @@ -83,7 +83,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -92,7 +92,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28789.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28788.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 2730 Data size: 32760 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) @@ -405,7 +405,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -414,7 +414,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28801.388)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28801.336)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 2730 Data size: 32760 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) diff --git a/ql/src/test/results/clientpositive/parquet_vectorization_6.q.out b/ql/src/test/results/clientpositive/parquet_vectorization_6.q.out index 0bb6888364..b35c11eaf1 100644 --- a/ql/src/test/results/clientpositive/parquet_vectorization_6.q.out +++ b/ql/src/test/results/clientpositive/parquet_vectorization_6.q.out @@ -58,7 +58,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257))))) (type: boolean) + filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0))))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -67,7 +67,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprAndExpr(children: FilterLongColNotEqualLongScalar(col 0:tinyint, val 0), FilterExprOrExpr(children: FilterExprAndExpr(children: FilterLongColLessEqualLongScalar(col 10:boolean, val 0), FilterLongColGreaterEqualLongColumn(col 11:boolean, col 10:boolean)), FilterExprAndExpr(children: SelectColumnIsNotNull(col 3:bigint), FilterExprOrExpr(children: FilterStringColLikeStringScalar(col 7:string, pattern %a), FilterDoubleColLessEqualDoubleScalar(col 4:float, val -257.0))))) - predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257)))) and (ctinyint <> 0Y)) (type: boolean) + predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0)))) and (ctinyint <> 0Y)) (type: boolean) Statistics: Num rows: 11605 Data size: 139260 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), cfloat (type: float), cstring1 (type: string), (988888 * UDFToInteger(csmallint)) (type: int), (- csmallint) (type: smallint), (- cfloat) (type: float), (-26.28D / UDFToDouble(cfloat)) (type: double), (cfloat * 359.0) (type: float), (cint % UDFToInteger(ctinyint)) (type: int), (- cdouble) (type: double), (UDFToInteger(ctinyint) - -75) (type: int), (762 * (cint % UDFToInteger(ctinyint))) (type: int) diff --git a/ql/src/test/results/clientpositive/parquet_vectorization_8.q.out b/ql/src/test/results/clientpositive/parquet_vectorization_8.q.out index 957bd7b264..e01cf4a8c0 100644 --- a/ql/src/test/results/clientpositive/parquet_vectorization_8.q.out +++ b/ql/src/test/results/clientpositive/parquet_vectorization_8.q.out @@ -66,7 +66,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D)) or (cfloat < -6432) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) + filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D)) or (cfloat < -6432.0) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -75,7 +75,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 10.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 16.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0))) - predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D))) (type: boolean) + predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432.0) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) @@ -272,7 +272,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D)) or (cfloat < -6432) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) + filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D)) or (cfloat < -6432.0) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -281,7 +281,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 12.503)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 11.998)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0))) - predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D))) (type: boolean) + predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432.0) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) diff --git a/ql/src/test/results/clientpositive/ppd_udf_col.q.out b/ql/src/test/results/clientpositive/ppd_udf_col.q.out index 814fb5afcf..9ef340d512 100644 --- a/ql/src/test/results/clientpositive/ppd_udf_col.q.out +++ b/ql/src/test/results/clientpositive/ppd_udf_col.q.out @@ -432,7 +432,7 @@ STAGE PLANS: outputColumnNames: _col0, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col2 <= 0.1) (type: boolean) + predicate: (_col2 <= 0.1D) (type: boolean) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: double) @@ -491,7 +491,7 @@ STAGE PLANS: outputColumnNames: _col0, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col2 <= 0.1) and (_col2 > 0.1)) (type: boolean) + predicate: ((_col2 <= 0.1D) and (_col2 > 0.1D)) (type: boolean) Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: double) @@ -644,7 +644,7 @@ STAGE PLANS: outputColumnNames: _col0, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col2 <= 0.1) (type: boolean) + predicate: (_col2 <= 0.1D) (type: boolean) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: double) @@ -703,14 +703,14 @@ STAGE PLANS: outputColumnNames: _col0, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col2 <= 0.1) (type: boolean) + predicate: (_col2 <= 0.1D) (type: boolean) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col1 > 0.1) (type: boolean) + predicate: (_col1 > 0.1D) (type: boolean) Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 20 diff --git a/ql/src/test/results/clientpositive/spark/cbo_simple_select.q.out b/ql/src/test/results/clientpositive/spark/cbo_simple_select.q.out index acf91bf178..91f393ee6b 100644 --- a/ql/src/test/results/clientpositive/spark/cbo_simple_select.q.out +++ b/ql/src/test/results/clientpositive/spark/cbo_simple_select.q.out @@ -833,13 +833,11 @@ PREHOOK: query: -- c_int is not null EXPLAIN select * from cbo_t2 where (cbo_t2.c_int) IN (NULL) PREHOOK: type: QUERY PREHOOK: Input: default@cbo_t2 -PREHOOK: Input: default@cbo_t2@dt=2014 #### A masked pattern was here #### POSTHOOK: query: -- c_int is not null EXPLAIN select * from cbo_t2 where (cbo_t2.c_int) IN (NULL) POSTHOOK: type: QUERY POSTHOOK: Input: default@cbo_t2 -POSTHOOK: Input: default@cbo_t2@dt=2014 #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-0 is a root stage @@ -851,11 +849,10 @@ STAGE PLANS: Processor Tree: TableScan alias: cbo_t2 - filterExpr: (c_int = null) (type: boolean) Filter Operator - predicate: (c_int = null) (type: boolean) + predicate: false (type: boolean) Select Operator - expressions: key (type: string), value (type: string), null (type: int), c_float (type: float), c_boolean (type: boolean), dt (type: string) + expressions: key (type: string), value (type: string), c_int (type: int), c_float (type: float), c_boolean (type: boolean), dt (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 ListSink @@ -964,13 +961,11 @@ PREHOOK: query: -- c_int is not null select count(*) from cbo_t2 where (cbo_t2.c_int) IN (NULL) PREHOOK: type: QUERY PREHOOK: Input: default@cbo_t2 -PREHOOK: Input: default@cbo_t2@dt=2014 #### A masked pattern was here #### POSTHOOK: query: -- c_int is not null select count(*) from cbo_t2 where (cbo_t2.c_int) IN (NULL) POSTHOOK: type: QUERY POSTHOOK: Input: default@cbo_t2 -POSTHOOK: Input: default@cbo_t2@dt=2014 #### A masked pattern was here #### 0 PREHOOK: query: -- rewrite to NULL diff --git a/ql/src/test/results/clientpositive/spark/parquet_vectorization_13.q.out b/ql/src/test/results/clientpositive/spark/parquet_vectorization_13.q.out index 3812239343..19880a820e 100644 --- a/ql/src/test/results/clientpositive/spark/parquet_vectorization_13.q.out +++ b/ql/src/test/results/clientpositive/spark/parquet_vectorization_13.q.out @@ -88,7 +88,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -97,7 +97,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28789.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28788.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 2730 Data size: 32760 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) @@ -421,7 +421,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -430,7 +430,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28801.388)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28801.336)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 2730 Data size: 32760 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) diff --git a/ql/src/test/results/clientpositive/spark/parquet_vectorization_6.q.out b/ql/src/test/results/clientpositive/spark/parquet_vectorization_6.q.out index 6108457aad..eef9f39e26 100644 --- a/ql/src/test/results/clientpositive/spark/parquet_vectorization_6.q.out +++ b/ql/src/test/results/clientpositive/spark/parquet_vectorization_6.q.out @@ -61,7 +61,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257))))) (type: boolean) + filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0))))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -70,7 +70,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprAndExpr(children: FilterLongColNotEqualLongScalar(col 0:tinyint, val 0), FilterExprOrExpr(children: FilterExprAndExpr(children: FilterLongColLessEqualLongScalar(col 10:boolean, val 0), FilterLongColGreaterEqualLongColumn(col 11:boolean, col 10:boolean)), FilterExprAndExpr(children: SelectColumnIsNotNull(col 3:bigint), FilterExprOrExpr(children: FilterStringColLikeStringScalar(col 7:string, pattern %a), FilterDoubleColLessEqualDoubleScalar(col 4:float, val -257.0))))) - predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257)))) and (ctinyint <> 0Y)) (type: boolean) + predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0)))) and (ctinyint <> 0Y)) (type: boolean) Statistics: Num rows: 11605 Data size: 139260 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), cfloat (type: float), cstring1 (type: string), (988888 * UDFToInteger(csmallint)) (type: int), (- csmallint) (type: smallint), (- cfloat) (type: float), (-26.28D / UDFToDouble(cfloat)) (type: double), (cfloat * 359.0) (type: float), (cint % UDFToInteger(ctinyint)) (type: int), (- cdouble) (type: double), (UDFToInteger(ctinyint) - -75) (type: int), (762 * (cint % UDFToInteger(ctinyint))) (type: int) diff --git a/ql/src/test/results/clientpositive/spark/parquet_vectorization_8.q.out b/ql/src/test/results/clientpositive/spark/parquet_vectorization_8.q.out index 3352dedc58..657e05cb09 100644 --- a/ql/src/test/results/clientpositive/spark/parquet_vectorization_8.q.out +++ b/ql/src/test/results/clientpositive/spark/parquet_vectorization_8.q.out @@ -71,7 +71,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D)) or (cfloat < -6432) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) + filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D)) or (cfloat < -6432.0) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -80,7 +80,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 10.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 16.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0))) - predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D))) (type: boolean) + predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432.0) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) @@ -295,7 +295,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesparquet - filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D)) or (cfloat < -6432) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) + filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D)) or (cfloat < -6432.0) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -304,7 +304,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 12.503)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 11.998)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0))) - predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D))) (type: boolean) + predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432.0) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D))) (type: boolean) Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) diff --git a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out index f5a4c9ad86..7477696c4e 100644 --- a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out @@ -452,7 +452,7 @@ Stage-0 Group By Operator [GBY_6] (rows=2 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_33] (rows=5 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 8 [PARTITION-LEVEL SORT] @@ -468,7 +468,7 @@ Stage-0 Group By Operator [GBY_13] (rows=2 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_34] (rows=5 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -544,7 +544,7 @@ Stage-0 Group By Operator [GBY_6] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_35] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 8 [PARTITION-LEVEL SORT] @@ -560,7 +560,7 @@ Stage-0 Group By Operator [GBY_13] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_36] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -629,7 +629,7 @@ Stage-0 Group By Operator [GBY_6] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_30] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 7 [PARTITION-LEVEL SORT] @@ -645,7 +645,7 @@ Stage-0 Group By Operator [GBY_13] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_31] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -719,7 +719,7 @@ Stage-0 Group By Operator [GBY_6] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_34] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 8 [PARTITION-LEVEL SORT] @@ -735,7 +735,7 @@ Stage-0 Group By Operator [GBY_13] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_35] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -804,7 +804,7 @@ Stage-0 Group By Operator [GBY_6] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_30] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 7 [PARTITION-LEVEL SORT] @@ -820,7 +820,7 @@ Stage-0 Group By Operator [GBY_13] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_31] (rows=2 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (c_float > 0) and key is not null) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (c_float > 0.0) and key is not null) TableScan [TS_10] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -1263,7 +1263,7 @@ Stage-0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1","_col2"] Filter Operator [FIL_17] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 3 [PARTITION-LEVEL SORT] @@ -1272,7 +1272,7 @@ Stage-0 Select Operator [SEL_5] (rows=9 width=89) Output:["_col0","_col1"] Filter Operator [FIL_18] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 4 [PARTITION-LEVEL SORT] @@ -1324,7 +1324,7 @@ Stage-0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1","_col2"] Filter Operator [FIL_17] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 3 [PARTITION-LEVEL SORT] @@ -1333,7 +1333,7 @@ Stage-0 Select Operator [SEL_5] (rows=9 width=89) Output:["_col0","_col1"] Filter Operator [FIL_18] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 4 [PARTITION-LEVEL SORT] @@ -1590,7 +1590,7 @@ Stage-0 Group By Operator [GBY_3] (rows=3 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_45] (rows=6 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0))) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) TableScan [TS_0] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 9 [PARTITION-LEVEL SORT] @@ -1614,7 +1614,7 @@ Stage-0 Group By Operator [GBY_16] (rows=3 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_46] (rows=6 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0))) + predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) TableScan [TS_13] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] @@ -1653,7 +1653,7 @@ Stage-0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1"] Filter Operator [FIL_13] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 3 [PARTITION-LEVEL SORT] @@ -1705,7 +1705,7 @@ Stage-0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1","_col2"] Filter Operator [FIL_19] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 3 [PARTITION-LEVEL SORT] @@ -1716,7 +1716,7 @@ Stage-0 Select Operator [SEL_5] (rows=9 width=85) Output:["_col0"] Filter Operator [FIL_20] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) + predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Map 4 [PARTITION-LEVEL SORT] @@ -1805,7 +1805,7 @@ Stage-0 Group By Operator [GBY_3] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float Filter Operator [FIL_35] (rows=1 width=93) - predicate:((((c_int + 1) + 1) >= 0) and (((c_int + 1) > 0) or (UDFToDouble(key) >= 0.0D)) and ((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (UDFToDouble(key) > 0.0D) and (c_float > 0)) + predicate:((((c_int + 1) + 1) >= 0) and (((c_int + 1) > 0) or (UDFToDouble(key) >= 0.0D)) and ((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (UDFToDouble(key) > 0.0D) and (c_float > 0.0)) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] <-Reducer 7 [PARTITION-LEVEL SORT] @@ -1823,7 +1823,7 @@ Stage-0 Group By Operator [GBY_12] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float Filter Operator [FIL_36] (rows=1 width=93) - predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1)) and (UDFToDouble(key) > 0.0D) and (c_float > 0)) + predicate:(((UDFToFloat(c_int) + c_float) >= 0) and ((c_int + 1) >= 0) and ((c_int > 0) or c_float is not null) and ((c_int >= 1) or (c_float >= 1.0)) and (UDFToDouble(key) > 0.0D) and (c_float > 0.0)) TableScan [TS_9] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] diff --git a/ql/src/test/results/clientpositive/spark/subquery_scalar.q.out b/ql/src/test/results/clientpositive/spark/subquery_scalar.q.out index b3252f5415..c20e92e282 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_scalar.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_scalar.q.out @@ -359,18 +359,17 @@ STAGE PLANS: Map Operator Tree: TableScan alias: part - filterExpr: (p_name = null) (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (p_name = null) (type: boolean) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: vectorized Map 3 @@ -420,14 +419,14 @@ STAGE PLANS: 1 2 outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 13 Data size: 424970 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 32690 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), null (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 13 Data size: 424970 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 32690 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 424970 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 32690 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/spark/vectorization_13.q.out b/ql/src/test/results/clientpositive/spark/vectorization_13.q.out index 34ec9c42dd..c25649bd14 100644 --- a/ql/src/test/results/clientpositive/spark/vectorization_13.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorization_13.q.out @@ -88,7 +88,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -98,7 +98,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28789.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28788.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 2730 Data size: 646063 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) @@ -445,7 +445,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -454,7 +454,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28801.388)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28801.336)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 2730 Data size: 646063 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) diff --git a/ql/src/test/results/clientpositive/spark/vectorization_6.q.out b/ql/src/test/results/clientpositive/spark/vectorization_6.q.out index 5679bb8cfa..d1cc6e39ed 100644 --- a/ql/src/test/results/clientpositive/spark/vectorization_6.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorization_6.q.out @@ -61,7 +61,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257))))) (type: boolean) + filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0))))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -71,7 +71,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprAndExpr(children: FilterLongColNotEqualLongScalar(col 0:tinyint, val 0), FilterExprOrExpr(children: FilterExprAndExpr(children: FilterLongColLessEqualLongScalar(col 10:boolean, val 0), FilterLongColGreaterEqualLongColumn(col 11:boolean, col 10:boolean)), FilterExprAndExpr(children: SelectColumnIsNotNull(col 3:bigint), FilterExprOrExpr(children: FilterStringColLikeStringScalar(col 7:string, pattern %a), FilterDoubleColLessEqualDoubleScalar(col 4:float, val -257.0))))) - predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257)))) and (ctinyint <> 0Y)) (type: boolean) + predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0)))) and (ctinyint <> 0Y)) (type: boolean) Statistics: Num rows: 11605 Data size: 2746359 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), cfloat (type: float), cstring1 (type: string), (988888 * UDFToInteger(csmallint)) (type: int), (- csmallint) (type: smallint), (- cfloat) (type: float), (-26.28D / UDFToDouble(cfloat)) (type: double), (cfloat * 359.0) (type: float), (cint % UDFToInteger(ctinyint)) (type: int), (- cdouble) (type: double), (UDFToInteger(ctinyint) - -75) (type: int), (762 * (cint % UDFToInteger(ctinyint))) (type: int) diff --git a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out index 231dea6de3..6c3ab0d851 100644 --- a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out @@ -617,7 +617,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((ctimestamp1 = ctimestamp2) or (cfloat = 762) or (cstring1 = 'ss') or ((UDFToLong(csmallint) <= cbigint) and (cboolean2 = 1)) or (cboolean1 is not null and ctimestamp2 is not null and (cstring2 > 'a'))) (type: boolean) + filterExpr: ((ctimestamp1 = ctimestamp2) or (cfloat = 762.0) or (cstring1 = 'ss') or ((UDFToLong(csmallint) <= cbigint) and (cboolean2 = 1)) or (cboolean1 is not null and ctimestamp2 is not null and (cstring2 > 'a'))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -626,7 +626,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterTimestampColEqualTimestampColumn(col 8:timestamp, col 9:timestamp), FilterDoubleColEqualDoubleScalar(col 4:float, val 762.0), FilterStringGroupColEqualStringScalar(col 6:string, val ss), FilterExprAndExpr(children: FilterLongColLessEqualLongColumn(col 1:bigint, col 3:bigint)(children: col 1:smallint), FilterLongColEqualLongScalar(col 11:boolean, val 1)), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), SelectColumnIsNotNull(col 9:timestamp), FilterStringGroupColGreaterStringScalar(col 7:string, val a))) - predicate: (((UDFToLong(csmallint) <= cbigint) and (cboolean2 = 1)) or (cboolean1 is not null and ctimestamp2 is not null and (cstring2 > 'a')) or (cfloat = 762) or (cstring1 = 'ss') or (ctimestamp1 = ctimestamp2)) (type: boolean) + predicate: (((UDFToLong(csmallint) <= cbigint) and (cboolean2 = 1)) or (cboolean1 is not null and ctimestamp2 is not null and (cstring2 > 'a')) or (cfloat = 762.0) or (cstring1 = 'ss') or (ctimestamp1 = ctimestamp2)) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cbigint (type: bigint), ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cdouble (type: double), UDFToDouble(cbigint) (type: double), (UDFToDouble(cbigint) * UDFToDouble(cbigint)) (type: double), UDFToDouble(csmallint) (type: double), (UDFToDouble(csmallint) * UDFToDouble(csmallint)) (type: double), (cdouble * cdouble) (type: double) @@ -853,7 +853,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: (((ctimestamp2 <= ctimestamp1) and (UDFToDouble(cbigint) <> cdouble) and (cstring1 >= 'ss')) or ((csmallint < UDFToShort(ctinyint)) and (UDFToDouble(ctimestamp1) >= 0.0D)) or (cfloat = 17)) (type: boolean) + filterExpr: (((ctimestamp2 <= ctimestamp1) and (UDFToDouble(cbigint) <> cdouble) and (cstring1 >= 'ss')) or ((csmallint < UDFToShort(ctinyint)) and (UDFToDouble(ctimestamp1) >= 0.0D)) or (cfloat = 17.0)) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -862,7 +862,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterTimestampColLessEqualTimestampColumn(col 9:timestamp, col 8:timestamp), FilterDoubleColNotEqualDoubleColumn(col 13:double, col 5:double)(children: CastLongToDouble(col 3:bigint) -> 13:double), FilterStringGroupColGreaterEqualStringScalar(col 6:string, val ss)), FilterExprAndExpr(children: FilterLongColLessLongColumn(col 1:smallint, col 0:smallint)(children: col 0:tinyint), FilterDoubleColGreaterEqualDoubleScalar(col 13:double, val 0.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double)), FilterDoubleColEqualDoubleScalar(col 4:float, val 17.0)) - predicate: (((csmallint < UDFToShort(ctinyint)) and (UDFToDouble(ctimestamp1) >= 0.0D)) or ((ctimestamp2 <= ctimestamp1) and (UDFToDouble(cbigint) <> cdouble) and (cstring1 >= 'ss')) or (cfloat = 17)) (type: boolean) + predicate: (((csmallint < UDFToShort(ctinyint)) and (UDFToDouble(ctimestamp1) >= 0.0D)) or ((ctimestamp2 <= ctimestamp1) and (UDFToDouble(cbigint) <> cdouble) and (cstring1 >= 'ss')) or (cfloat = 17.0)) (type: boolean) Statistics: Num rows: 8874 Data size: 2100060 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctinyint (type: tinyint), cbigint (type: bigint), cint (type: int), cfloat (type: float), UDFToDouble(cint) (type: double), (UDFToDouble(cint) * UDFToDouble(cint)) (type: double), UDFToDouble(cbigint) (type: double), (UDFToDouble(cbigint) * UDFToDouble(cbigint)) (type: double) diff --git a/ql/src/test/results/clientpositive/vectorization_13.q.out b/ql/src/test/results/clientpositive/vectorization_13.q.out index 8897f8427f..2ed66189e7 100644 --- a/ql/src/test/results/clientpositive/vectorization_13.q.out +++ b/ql/src/test/results/clientpositive/vectorization_13.q.out @@ -83,7 +83,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -93,7 +93,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28789.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28788.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 2730 Data size: 646063 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) @@ -419,7 +419,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: (((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) + filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -428,7 +428,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28801.388)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val -28801.336)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDecimalColLessDecimalScalar(col 14:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(11,4)))) - predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) + predicate: (((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639)) or ((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1))) (type: boolean) Statistics: Num rows: 2730 Data size: 646063 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double) diff --git a/ql/src/test/results/clientpositive/vectorization_6.q.out b/ql/src/test/results/clientpositive/vectorization_6.q.out index 8dedb63e7d..c33c9bb9f0 100644 --- a/ql/src/test/results/clientpositive/vectorization_6.q.out +++ b/ql/src/test/results/clientpositive/vectorization_6.q.out @@ -58,7 +58,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257))))) (type: boolean) + filterExpr: ((ctinyint <> 0Y) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0))))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -68,7 +68,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprAndExpr(children: FilterLongColNotEqualLongScalar(col 0:tinyint, val 0), FilterExprOrExpr(children: FilterExprAndExpr(children: FilterLongColLessEqualLongScalar(col 10:boolean, val 0), FilterLongColGreaterEqualLongColumn(col 11:boolean, col 10:boolean)), FilterExprAndExpr(children: SelectColumnIsNotNull(col 3:bigint), FilterExprOrExpr(children: FilterStringColLikeStringScalar(col 7:string, pattern %a), FilterDoubleColLessEqualDoubleScalar(col 4:float, val -257.0))))) - predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257)))) and (ctinyint <> 0Y)) (type: boolean) + predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257.0)))) and (ctinyint <> 0Y)) (type: boolean) Statistics: Num rows: 11605 Data size: 2746359 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), cfloat (type: float), cstring1 (type: string), (988888 * UDFToInteger(csmallint)) (type: int), (- csmallint) (type: smallint), (- cfloat) (type: float), (-26.28D / UDFToDouble(cfloat)) (type: double), (cfloat * 359.0) (type: float), (cint % UDFToInteger(ctinyint)) (type: int), (- cdouble) (type: double), (UDFToInteger(ctinyint) - -75) (type: int), (762 * (cint % UDFToInteger(ctinyint))) (type: int) diff --git a/ql/src/test/results/clientpositive/vectorization_8.q.out b/ql/src/test/results/clientpositive/vectorization_8.q.out index d81df76a2f..498506cbb7 100644 --- a/ql/src/test/results/clientpositive/vectorization_8.q.out +++ b/ql/src/test/results/clientpositive/vectorization_8.q.out @@ -66,7 +66,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D)) or (cfloat < -6432) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) + filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D)) or (cfloat < -6432.0) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -76,7 +76,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 10.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 16.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0))) - predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D))) (type: boolean) + predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432.0) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) @@ -279,7 +279,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D)) or (cfloat < -6432) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) + filterExpr: ((cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D)) or (cfloat < -6432.0) or (cboolean1 is not null and (cdouble = 988888.0D))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -288,7 +288,7 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 12.503)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 11.998)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0))) - predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D))) (type: boolean) + predicate: ((cboolean1 is not null and (cdouble = 988888.0D)) or (cfloat < -6432.0) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D))) (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double)