diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index 14a1059..01ed31d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -281,7 +281,7 @@ protected int getInputColumnIndex(String name) throws HiveException { throw new HiveException("Null column name"); } if (!projectionColumnMap.containsKey(name)) { - throw new HiveException(String.format("The column %s is not in the vectorization context column map %s.", + throw new HiveException(String.format("The column %s is not in the vectorization context column map %s.", name, projectionColumnMap.toString())); } return projectionColumnMap.get(name); @@ -1006,7 +1006,7 @@ private VectorExpression createVectorExpression(Class vectorClass, arguments[i] = colIndex; } else if (child instanceof ExprNodeConstantDesc) { Object scalarValue = getVectorTypeScalarValue((ExprNodeConstantDesc) child); - arguments[i] = scalarValue; + arguments[i] = (null == scalarValue) ? getConstantVectorExpression(null, child.getTypeInfo(), childrenMode) : scalarValue; } else { throw new HiveException("Cannot handle expression type: " + child.getClass().getSimpleName()); } @@ -1321,10 +1321,10 @@ private HiveDecimal castConstantToDecimal(Object scalar, TypeInfo type) throws H HiveDecimal rawDecimal; switch (ptinfo.getPrimitiveCategory()) { case FLOAT: - rawDecimal = HiveDecimal.create(String.valueOf((Float) scalar)); + rawDecimal = HiveDecimal.create(String.valueOf(scalar)); break; case DOUBLE: - rawDecimal = HiveDecimal.create(String.valueOf((Double) scalar)); + rawDecimal = HiveDecimal.create(String.valueOf(scalar)); break; case BYTE: rawDecimal = HiveDecimal.create((Byte) scalar); @@ -1488,7 +1488,7 @@ private VectorExpression getCastToChar(List childExpr, TypeInfo re return createVectorExpression(CastStringGroupToChar.class, childExpr, Mode.PROJECTION, returnType); } - /* + /* * Timestamp, float, and double types are handled by the legacy code path. See isLegacyPathUDF. */ @@ -1836,7 +1836,7 @@ private Object getScalarValue(ExprNodeConstantDesc constDesc) return 0; } } else if (decimalTypePattern.matcher(constDesc.getTypeString()).matches()) { - return (HiveDecimal) constDesc.getValue(); + return constDesc.getValue(); } else { return constDesc.getValue(); } @@ -1976,7 +1976,7 @@ static String getNormalizedName(String hiveTypeName) { return "None"; } } - + static String getUndecoratedName(String hiveTypeName) { VectorExpressionDescriptor.ArgumentType argType = VectorExpressionDescriptor.ArgumentType.fromHiveTypeName(hiveTypeName); switch (argType) { @@ -2005,7 +2005,7 @@ static String getUndecoratedName(String hiveTypeName) { } // TODO: When we support vectorized STRUCTs and can handle more in the reduce-side (MERGEPARTIAL): - // TODO: Write reduce-side versions of AVG. Currently, only map-side (HASH) versions are in table. + // TODO: Write reduce-side versions of AVG. Currently, only map-side (HASH) versions are in table. // TODO: And, investigate if different reduce-side versions are needed for var* and std*, or if map-side aggregate can be used.. Right now they are conservatively // marked map-side (HASH). static ArrayList aggregatesDefinition = new ArrayList() {{ @@ -2115,6 +2115,7 @@ public VectorAggregateExpression getAggregatorExpression(AggregationDesc desc, b return map; } + @Override public String toString() { StringBuilder sb = new StringBuilder(32); sb.append("Context key ").append(getFileKey()).append(", "); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java index c76b15b..8d75cf3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java @@ -18,13 +18,10 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; -import java.util.Arrays; - import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; /** * Constant is represented as a vector with repeating values. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java index b0768f2..e9436e5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java @@ -488,9 +488,8 @@ private static ExprNodeDesc evaluateFunction(GenericUDF udf, List ObjectInspectorUtils.getConstantObjectInspector(constant.getWritableObjectInspector(), writableValue); } else if (desc instanceof ExprNodeNullDesc) { - - // FIXME: add null support. - return null; + argois[i] = desc.getWritableObjectInspector(); + arguments[i] = new DeferredJavaObject(((ExprNodeNullDesc) desc).getValue()); } else if (desc instanceof ExprNodeGenericFuncDesc) { ExprNodeDesc evaluatedFn = foldExpr((ExprNodeGenericFuncDesc)desc); if (null == evaluatedFn || !(evaluatedFn instanceof ExprNodeConstantDesc)) { @@ -511,6 +510,10 @@ private static ExprNodeDesc evaluateFunction(GenericUDF udf, List Object o = udf.evaluate(arguments); LOG.debug(udf.getClass().getName() + "(" + exprs + ")=" + o); if (o == null) { + if (oi instanceof PrimitiveObjectInspector) { + + return new ExprNodeConstantDesc(((PrimitiveObjectInspector) oi).getTypeInfo(), o); + } return new ExprNodeNullDesc(); } Class clz = o.getClass(); @@ -604,6 +607,10 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, Object.. LOG.warn("Filter expression " + condn + " holds false!"); } } + if (newCondn instanceof ExprNodeNullDesc || (newCondn instanceof ExprNodeConstantDesc && ((ExprNodeConstantDesc)newCondn).getValue() == null)) { + // where null is same as where false + newCondn = new ExprNodeConstantDesc(Boolean.FALSE); + } LOG.debug("New filter FIL[" + op.getIdentifier() + "] conditions:" + newCondn.getExprString()); // merge it with the downstream col list diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java index d18e1a7..44269f0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java @@ -352,7 +352,7 @@ private long evaluateExpression(Statistics stats, ExprNodeDesc pred, // special case for handling false constants ExprNodeConstantDesc encd = (ExprNodeConstantDesc) pred; - if (encd.getValue().equals(false)) { + if (Boolean.FALSE.equals(encd.getValue())) { return 0; } else { return stats.getNumRows(); @@ -383,7 +383,7 @@ private long evaluateNotExpr(Statistics stats, ExprNodeDesc pred, return numRows - newNumRows; } else if (leaf instanceof ExprNodeConstantDesc) { ExprNodeConstantDesc encd = (ExprNodeConstantDesc) leaf; - if (encd.getValue().equals(true)) { + if (Boolean.TRUE.equals(encd.getValue())) { return 0; } else { return numRows; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java index 513d030..58c2f44 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java @@ -82,7 +82,7 @@ * Object reference. */ public static class DeferredJavaObject implements DeferredObject { - private Object value; + private final Object value; public DeferredJavaObject(Object value) { this.value = value; @@ -208,6 +208,7 @@ public abstract Object evaluate(DeferredObject[] arguments) * Close GenericUDF. * This is only called in runtime of MapRedTask. */ + @Override public void close() throws IOException { } @@ -314,7 +315,7 @@ protected void obtainStringConverter(ObjectInspector[] arguments, int i, PrimitiveCategory inputType = inOi.getPrimitiveCategory(); Converter converter = ObjectInspectorConverters.getConverter( - (PrimitiveObjectInspector) arguments[i], + arguments[i], PrimitiveObjectInspectorFactory.writableStringObjectInspector); converters[i] = converter; inputTypes[i] = inputType; @@ -328,6 +329,7 @@ protected void obtainIntConverter(ObjectInspector[] arguments, int i, case BYTE: case SHORT: case INT: + case VOID: break; default: throw new UDFArgumentTypeException(i, getFuncName() + " only takes INT/SHORT/BYTE types as " @@ -335,7 +337,7 @@ protected void obtainIntConverter(ObjectInspector[] arguments, int i, } Converter converter = ObjectInspectorConverters.getConverter( - (PrimitiveObjectInspector) arguments[i], + arguments[i], PrimitiveObjectInspectorFactory.writableIntObjectInspector); converters[i] = converter; inputTypes[i] = inputType; @@ -358,7 +360,7 @@ protected void obtainLongConverter(ObjectInspector[] arguments, int i, } Converter converter = ObjectInspectorConverters.getConverter( - (PrimitiveObjectInspector) arguments[i], + arguments[i], PrimitiveObjectInspectorFactory.writableIntObjectInspector); converters[i] = converter; inputTypes[i] = inputType; @@ -369,7 +371,7 @@ protected void obtainDoubleConverter(ObjectInspector[] arguments, int i, PrimitiveObjectInspector inOi = (PrimitiveObjectInspector) arguments[i]; PrimitiveCategory inputType = inOi.getPrimitiveCategory(); Converter converter = ObjectInspectorConverters.getConverter( - (PrimitiveObjectInspector) arguments[i], + arguments[i], PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); converters[i] = converter; inputTypes[i] = inputType; @@ -388,6 +390,7 @@ protected void obtainDateConverter(ObjectInspector[] arguments, int i, break; case TIMESTAMP: case DATE: + case VOID: outOi = PrimitiveObjectInspectorFactory.writableDateObjectInspector; break; default: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAddMonths.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAddMonths.java index 4cf6318..82e5811 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAddMonths.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAddMonths.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.DATE_GROUP; import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.NUMERIC_GROUP; import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.STRING_GROUP; +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.VOID_GROUP; import java.util.Calendar; import java.util.Date; @@ -63,8 +64,8 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen checkArgPrimitive(arguments, 0); checkArgPrimitive(arguments, 1); - checkArgGroups(arguments, 0, inputTypes, STRING_GROUP, DATE_GROUP); - checkArgGroups(arguments, 1, inputTypes, NUMERIC_GROUP); + checkArgGroups(arguments, 0, inputTypes, STRING_GROUP, DATE_GROUP, VOID_GROUP); + checkArgGroups(arguments, 1, inputTypes, NUMERIC_GROUP, VOID_GROUP); obtainDateConverter(arguments, 0, inputTypes, converters); obtainIntConverter(arguments, 1, inputTypes, converters); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArray.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArray.java index 55a47fb..ee1c8d1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArray.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArray.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector; /** * GenericUDFArray. @@ -43,9 +44,7 @@ @Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { - GenericUDFUtils.ReturnObjectInspectorResolver returnOIResolver; - - returnOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true); + GenericUDFUtils.ReturnObjectInspectorResolver returnOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true); for (int i = 0; i < arguments.length; i++) { if (!returnOIResolver.update(arguments[i])) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCbrt.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCbrt.java index 732ce8a..1cbbaa7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCbrt.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCbrt.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf.generic; import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.NUMERIC_GROUP; +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.VOID_GROUP; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; @@ -48,7 +49,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen checkArgPrimitive(arguments, 0); - checkArgGroups(arguments, 0, inputTypes, NUMERIC_GROUP); + checkArgGroups(arguments, 0, inputTypes, NUMERIC_GROUP, VOID_GROUP); obtainDoubleConverter(arguments, 0, inputTypes, converters); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDecode.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDecode.java index 9858b4f..f4366c7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDecode.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDecode.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; @Description(name = "decode", @@ -48,7 +49,7 @@ "is null, the result will also be null") public class GenericUDFDecode extends GenericUDF { private transient CharsetDecoder decoder; - private transient BinaryObjectInspector bytesOI; + private transient PrimitiveObjectInspector bytesOI; private transient PrimitiveObjectInspector charsetOI; @Override @@ -57,12 +58,19 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen throw new UDFArgumentLengthException("Decode() requires exactly two arguments"); } - if (arguments[0].getCategory() != Category.PRIMITIVE || - ((PrimitiveObjectInspector)arguments[0]).getPrimitiveCategory() != PrimitiveCategory.BINARY){ - throw new UDFArgumentTypeException(0, "The first argument to Decode() must be a binary"); + if (arguments[0].getCategory() != Category.PRIMITIVE){ + throw new UDFArgumentTypeException(0, "The first argument to Decode() must be primitive"); } - bytesOI = (BinaryObjectInspector) arguments[0]; + PrimitiveCategory category = ((PrimitiveObjectInspector)arguments[0]).getPrimitiveCategory(); + + if (category == PrimitiveCategory.BINARY) { + bytesOI = (BinaryObjectInspector) arguments[0]; + } else if(category == PrimitiveCategory.VOID) { + bytesOI = (VoidObjectInspector) arguments[0]; + } else { + throw new UDFArgumentTypeException(0, "The first argument to Decode() must be binary"); + } if (arguments[1].getCategory() != Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "The second argument to Decode() must be primitive"); @@ -85,17 +93,17 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen .onUnmappableCharacter(CodingErrorAction.REPORT); } - return (ObjectInspector) PrimitiveObjectInspectorFactory.javaStringObjectInspector; + return PrimitiveObjectInspectorFactory.javaStringObjectInspector; } @Override public Object evaluate(DeferredObject[] arguments) throws HiveException { - byte[] value = bytesOI.getPrimitiveJavaObject(arguments[0].get()); + Object value = bytesOI.getPrimitiveJavaObject(arguments[0].get()); if (value == null) { return null; } - ByteBuffer wrappedBytes = ByteBuffer.wrap(value); + ByteBuffer wrappedBytes = ByteBuffer.wrap((byte[])value); CharBuffer decoded; if (decoder != null){ try { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFactorial.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFactorial.java index ff63b1d..4aa136a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFactorial.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFactorial.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf.generic; import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.NUMERIC_GROUP; +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.VOID_GROUP; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; @@ -60,7 +61,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen checkArgPrimitive(arguments, 0); - checkArgGroups(arguments, 0, inputTypes, NUMERIC_GROUP); + checkArgGroups(arguments, 0, inputTypes, NUMERIC_GROUP, VOID_GROUP); obtainIntConverter(arguments, 0, inputTypes, converters); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLastDay.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLastDay.java index 6ead4be..72e1e5a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLastDay.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLastDay.java @@ -19,7 +19,7 @@ import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.DATE_GROUP; import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.STRING_GROUP; - +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.VOID_GROUP; import java.util.Calendar; import java.util.Date; @@ -57,7 +57,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen checkArgPrimitive(arguments, 0); - checkArgGroups(arguments, 0, inputTypes, STRING_GROUP, DATE_GROUP); + checkArgGroups(arguments, 0, inputTypes, STRING_GROUP, DATE_GROUP, VOID_GROUP); obtainDateConverter(arguments, 0, inputTypes, converters); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLevenshtein.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLevenshtein.java index 4bba876..58941d8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLevenshtein.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLevenshtein.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.udf.generic; import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.STRING_GROUP; +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.VOID_GROUP; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.ql.exec.Description; @@ -60,8 +61,8 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen checkArgPrimitive(arguments, 0); checkArgPrimitive(arguments, 1); - checkArgGroups(arguments, 0, inputTypes, STRING_GROUP); - checkArgGroups(arguments, 1, inputTypes, STRING_GROUP); + checkArgGroups(arguments, 0, inputTypes, STRING_GROUP, VOID_GROUP); + checkArgGroups(arguments, 1, inputTypes, STRING_GROUP, VOID_GROUP); obtainStringConverter(arguments, 0, inputTypes, converters); obtainStringConverter(arguments, 1, inputTypes, converters); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMap.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMap.java index 4234b76..78318f4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMap.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMap.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.ql.udf.generic; -import java.util.HashMap; import java.util.LinkedHashMap; import org.apache.hadoop.hive.ql.exec.Description; @@ -26,12 +25,14 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector; /** * GenericUDFMap. @@ -75,7 +76,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen } } else { // Values - if (!valueOIResolver.update(arguments[i])) { + if (!valueOIResolver.update(arguments[i]) && !compatibleTypes(arguments[i], arguments[i-2])) { throw new UDFArgumentTypeException(i, "Value type \"" + arguments[i].getTypeName() + "\" is different from preceding value types. " @@ -100,6 +101,21 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen return ObjectInspectorFactory.getStandardMapObjectInspector(keyOI, valueOI); } + private boolean compatibleTypes(ObjectInspector current, ObjectInspector prev) { + + if (current instanceof VoidObjectInspector || prev instanceof VoidObjectInspector) { + // we allow null values for map. + return true; + } + if (current instanceof ListObjectInspector && prev instanceof ListObjectInspector && ( + ((ListObjectInspector)current).getListElementObjectInspector() instanceof VoidObjectInspector || + ((ListObjectInspector)prev).getListElementObjectInspector() instanceof VoidObjectInspector)) { + // array is compatible with any other array + return true; + } + return false; + } + @Override public Object evaluate(DeferredObject[] arguments) throws HiveException { ret.clear(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNextDay.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNextDay.java index c0a0ab1..67b4c54 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNextDay.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNextDay.java @@ -26,6 +26,7 @@ import static org.apache.hadoop.hive.ql.udf.generic.GenericUDFNextDay.DayOfWeek.WED; import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.DATE_GROUP; import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.STRING_GROUP; +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.VOID_GROUP; import java.util.Calendar; import java.util.Date; @@ -68,8 +69,8 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen checkArgPrimitive(arguments, 0); checkArgPrimitive(arguments, 1); - checkArgGroups(arguments, 0, inputTypes, STRING_GROUP, DATE_GROUP); - checkArgGroups(arguments, 1, inputTypes, STRING_GROUP); + checkArgGroups(arguments, 0, inputTypes, STRING_GROUP, DATE_GROUP, VOID_GROUP); + checkArgGroups(arguments, 1, inputTypes, STRING_GROUP, VOID_GROUP); obtainDateConverter(arguments, 0, inputTypes, converters); obtainStringConverter(arguments, 1, inputTypes, converters); @@ -168,9 +169,9 @@ protected int getIntDayOfWeek(String dayOfWeek) throws UDFArgumentException { "TH", "THU", "THURSDAY"), FRI("FR", "FRI", "FRIDAY"), SAT("SA", "SAT", "SATURDAY"), SUN( "SU", "SUN", "SUNDAY"); - private String name2; - private String name3; - private String fullName; + private final String name2; + private final String name3; + private final String fullName; private DayOfWeek(String name2, String name3, String fullName) { this.name2 = name2; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSoundex.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSoundex.java index ad72d05..be30b6a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSoundex.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSoundex.java @@ -100,7 +100,8 @@ protected void checkIfPrimitive(ObjectInspector[] arguments, int i, String argOr protected void checkIfStringGroup(ObjectInspector[] arguments, int i, String argOrder) throws UDFArgumentTypeException { inputTypes[i] = ((PrimitiveObjectInspector) arguments[i]).getPrimitiveCategory(); - if (PrimitiveObjectInspectorUtils.getPrimitiveGrouping(inputTypes[i]) != PrimitiveGrouping.STRING_GROUP) { + if (PrimitiveObjectInspectorUtils.getPrimitiveGrouping(inputTypes[i]) != PrimitiveGrouping.STRING_GROUP && + PrimitiveObjectInspectorUtils.getPrimitiveGrouping(inputTypes[i]) != PrimitiveGrouping.VOID_GROUP) { throw new UDFArgumentTypeException(i, getFuncName() + " only takes STRING_GROUP types as " + argOrder + " argument, got " + inputTypes[i]); } @@ -109,10 +110,11 @@ protected void checkIfStringGroup(ObjectInspector[] arguments, int i, String arg protected void getStringConverter(ObjectInspector[] arguments, int i, String argOrder) throws UDFArgumentTypeException { textConverters[i] = ObjectInspectorConverters.getConverter( - (PrimitiveObjectInspector) arguments[i], + arguments[i], PrimitiveObjectInspectorFactory.writableStringObjectInspector); } + @Override protected String getFuncName() { return "soundex"; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTrunc.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTrunc.java index 3bf8d34..e20ad65 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTrunc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTrunc.java @@ -22,7 +22,6 @@ import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.ArrayList; import java.util.Calendar; import java.util.Date; @@ -96,9 +95,10 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen case STRING: case VARCHAR: case CHAR: + case VOID: inputType1 = PrimitiveCategory.STRING; textConverter1 = ObjectInspectorConverters.getConverter( - (PrimitiveObjectInspector) arguments[0], + arguments[0], PrimitiveObjectInspectorFactory.writableStringObjectInspector); break; case TIMESTAMP: @@ -107,7 +107,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen break; case DATE: dateWritableConverter = ObjectInspectorConverters.getConverter( - (PrimitiveObjectInspector) arguments[0], + arguments[0], PrimitiveObjectInspectorFactory.writableDateObjectInspector); break; default: @@ -118,7 +118,8 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen inputType2 = ((PrimitiveObjectInspector) arguments[1]).getPrimitiveCategory(); if (PrimitiveObjectInspectorUtils.getPrimitiveGrouping(inputType2) - != PrimitiveGrouping.STRING_GROUP) { + != PrimitiveGrouping.STRING_GROUP && PrimitiveObjectInspectorUtils.getPrimitiveGrouping(inputType2) + != PrimitiveGrouping.VOID_GROUP) { throw new UDFArgumentTypeException(1, "trunk() only takes STRING/CHAR/VARCHAR types as second argument, got " + inputType2); } @@ -130,7 +131,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen fmtInput = obj != null ? obj.toString() : null; } else { textConverter2 = ObjectInspectorConverters.getConverter( - (PrimitiveObjectInspector) arguments[1], + arguments[1], PrimitiveObjectInspectorFactory.writableStringObjectInspector); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFLevenshtein.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFLevenshtein.java index 9f14ffd..8068cd4 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFLevenshtein.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFLevenshtein.java @@ -62,7 +62,7 @@ public void testLevenshteinWrongType0() throws HiveException { assertTrue("levenshtein test. UDFArgumentTypeException is expected", false); } catch (UDFArgumentTypeException e) { assertEquals("levenshtein test", - "levenshtein only takes STRING_GROUP types as 1st argument, got INT", e.getMessage()); + "levenshtein only takes STRING_GROUP, VOID_GROUP types as 1st argument, got INT", e.getMessage()); } } @@ -78,7 +78,7 @@ public void testLevenshteinWrongType1() throws HiveException { assertTrue("levenshtein test. UDFArgumentTypeException is expected", false); } catch (UDFArgumentTypeException e) { assertEquals("levenshtein test", - "levenshtein only takes STRING_GROUP types as 2nd argument, got FLOAT", e.getMessage()); + "levenshtein only takes STRING_GROUP, VOID_GROUP types as 2nd argument, got FLOAT", e.getMessage()); } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFNextDay.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFNextDay.java index 83ded3c..0556c2b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFNextDay.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFNextDay.java @@ -92,7 +92,7 @@ public void testNextDayErrorArg1() throws HiveException { assertTrue("UDFArgumentException expected", false); } catch (UDFArgumentException e) { assertEquals( - "next_day only takes STRING_GROUP, DATE_GROUP types as 1st argument, got LONG", + "next_day only takes STRING_GROUP, DATE_GROUP, VOID_GROUP types as 1st argument, got LONG", e.getMessage()); } } @@ -108,7 +108,7 @@ public void testNextDayErrorArg2() throws HiveException { udf.initialize(arguments); assertTrue("UDFArgumentException expected", false); } catch (UDFArgumentException e) { - assertEquals("next_day only takes STRING_GROUP types as 2nd argument, got INT", + assertEquals("next_day only takes STRING_GROUP, VOID_GROUP types as 2nd argument, got INT", e.getMessage()); } } diff --git a/ql/src/test/queries/clientpositive/optimize_nullscan.q b/ql/src/test/queries/clientpositive/optimize_nullscan.q index 8e2ae04..89293c5 100644 --- a/ql/src/test/queries/clientpositive/optimize_nullscan.q +++ b/ql/src/test/queries/clientpositive/optimize_nullscan.q @@ -31,3 +31,7 @@ select * from (select key from src union all select src.key from src left outer explain extended select * from src s1, src s2 where false and s1.value = s2.value; select * from src s1, src s2 where false and s1.value = s2.value; + +explain extended +select count(1) from src where null = 1; +select count(1) from src where null = 1; diff --git a/ql/src/test/results/clientnegative/udf_add_months_error_1.q.out b/ql/src/test/results/clientnegative/udf_add_months_error_1.q.out index e128612..8050bf7 100644 --- a/ql/src/test/results/clientnegative/udf_add_months_error_1.q.out +++ b/ql/src/test/results/clientnegative/udf_add_months_error_1.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10016]: Line 1:18 Argument type mismatch '14567893456': add_months only takes STRING_GROUP, DATE_GROUP types as 1st argument, got LONG +FAILED: SemanticException [Error 10016]: Line 1:18 Argument type mismatch '14567893456': add_months only takes STRING_GROUP, DATE_GROUP, VOID_GROUP types as 1st argument, got LONG diff --git a/ql/src/test/results/clientnegative/udf_last_day_error_1.q.out b/ql/src/test/results/clientnegative/udf_last_day_error_1.q.out index 71376e2..3ebe3a9 100644 --- a/ql/src/test/results/clientnegative/udf_last_day_error_1.q.out +++ b/ql/src/test/results/clientnegative/udf_last_day_error_1.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10016]: Line 1:16 Argument type mismatch '1423199465': last_day only takes STRING_GROUP, DATE_GROUP types as 1st argument, got INT +FAILED: SemanticException [Error 10016]: Line 1:16 Argument type mismatch '1423199465': last_day only takes STRING_GROUP, DATE_GROUP, VOID_GROUP types as 1st argument, got INT diff --git a/ql/src/test/results/clientnegative/udf_next_day_error_1.q.out b/ql/src/test/results/clientnegative/udf_next_day_error_1.q.out index 1d9c25f..a099b87 100644 --- a/ql/src/test/results/clientnegative/udf_next_day_error_1.q.out +++ b/ql/src/test/results/clientnegative/udf_next_day_error_1.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10016]: Line 1:16 Argument type mismatch '145622345': next_day only takes STRING_GROUP, DATE_GROUP types as 1st argument, got INT +FAILED: SemanticException [Error 10016]: Line 1:16 Argument type mismatch '145622345': next_day only takes STRING_GROUP, DATE_GROUP, VOID_GROUP types as 1st argument, got INT diff --git a/ql/src/test/results/clientnegative/udf_next_day_error_2.q.out b/ql/src/test/results/clientnegative/udf_next_day_error_2.q.out index e23186a..7fd61ab 100644 --- a/ql/src/test/results/clientnegative/udf_next_day_error_2.q.out +++ b/ql/src/test/results/clientnegative/udf_next_day_error_2.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10016]: Line 1:30 Argument type mismatch '4': next_day only takes STRING_GROUP types as 2nd argument, got INT +FAILED: SemanticException [Error 10016]: Line 1:30 Argument type mismatch '4': next_day only takes STRING_GROUP, VOID_GROUP types as 2nd argument, got INT diff --git a/ql/src/test/results/clientpositive/annotate_stats_select.q.out b/ql/src/test/results/clientpositive/annotate_stats_select.q.out index 49c1a40..753ab4e 100644 --- a/ql/src/test/results/clientpositive/annotate_stats_select.q.out +++ b/ql/src/test/results/clientpositive/annotate_stats_select.q.out @@ -508,7 +508,7 @@ STAGE PLANS: alias: alltypes_orc Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: null (type: void) + expressions: null (type: date) outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE ListSink diff --git a/ql/src/test/results/clientpositive/decimal_udf.q.out b/ql/src/test/results/clientpositive/decimal_udf.q.out index 59b5643..f36670c 100644 --- a/ql/src/test/results/clientpositive/decimal_udf.q.out +++ b/ql/src/test/results/clientpositive/decimal_udf.q.out @@ -940,7 +940,7 @@ STAGE PLANS: alias: decimal_udf Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: (UDFToDouble(key) / UDFToDouble(null)) (type: double) + expressions: (UDFToDouble(key) / null) (type: double) outputColumnNames: _col0 Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE Limit diff --git a/ql/src/test/results/clientpositive/input8.q.out b/ql/src/test/results/clientpositive/input8.q.out index 47bac2c..03857fc 100644 --- a/ql/src/test/results/clientpositive/input8.q.out +++ b/ql/src/test/results/clientpositive/input8.q.out @@ -32,7 +32,7 @@ STAGE PLANS: alias: src1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: (4.0 + UDFToDouble(null)) (type: double), UDFToInteger((UDFToDouble(key) - UDFToDouble(null))) (type: int), (UDFToDouble(null) + UDFToDouble(null)) (type: double) + expressions: null (type: double), UDFToInteger((UDFToDouble(key) - null)) (type: int), null (type: double) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/input9.q.out b/ql/src/test/results/clientpositive/input9.q.out index 4666787..d1824e5 100644 --- a/ql/src/test/results/clientpositive/input9.q.out +++ b/ql/src/test/results/clientpositive/input9.q.out @@ -32,15 +32,15 @@ STAGE PLANS: alias: src1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (null = null) (type: boolean) - Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: null (type: void), UDFToInteger(key) (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/load_dyn_part14.q.out b/ql/src/test/results/clientpositive/load_dyn_part14.q.out index ccf6f82..5af8490 100644 --- a/ql/src/test/results/clientpositive/load_dyn_part14.q.out +++ b/ql/src/test/results/clientpositive/load_dyn_part14.q.out @@ -74,24 +74,24 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'k1' (type: string), UDFToString(null) (type: string) + expressions: 'k1' (type: string), null (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 85000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 2 - Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 2 - Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false table: @@ -104,10 +104,10 @@ STAGE PLANS: Map Operator Tree: TableScan Union - Statistics: Num rows: 6 Data size: 1022 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 854 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 1022 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 854 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -115,10 +115,10 @@ STAGE PLANS: name: default.nzhang_part14 TableScan Union - Statistics: Num rows: 6 Data size: 1022 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 854 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 1022 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 854 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -126,10 +126,10 @@ STAGE PLANS: name: default.nzhang_part14 TableScan Union - Statistics: Num rows: 6 Data size: 1022 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 854 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 1022 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 854 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/num_op_type_conv.q.out b/ql/src/test/results/clientpositive/num_op_type_conv.q.out index 708fb51..013a153 100644 --- a/ql/src/test/results/clientpositive/num_op_type_conv.q.out +++ b/ql/src/test/results/clientpositive/num_op_type_conv.q.out @@ -20,12 +20,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: (UDFToDouble(null) + 7.0) (type: double), (1.0 - UDFToDouble(null)) (type: double), (UDFToDouble(null) + UDFToDouble(null)) (type: double), 1 (type: bigint), 0 (type: bigint), 0.0 (type: double) + expressions: null (type: double), null (type: double), null (type: double), 1 (type: bigint), 0 (type: bigint), 0.0 (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 500 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 12000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 - Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT null + 7, 1.0 - null, null + null, diff --git a/ql/src/test/results/clientpositive/optimize_nullscan.q.out b/ql/src/test/results/clientpositive/optimize_nullscan.q.out index 609e415..8e6376b 100644 --- a/ql/src/test/results/clientpositive/optimize_nullscan.q.out +++ b/ql/src/test/results/clientpositive/optimize_nullscan.q.out @@ -2156,3 +2156,150 @@ POSTHOOK: query: select * from src s1, src s2 where false and s1.value = s2.valu POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +PREHOOK: query: explain extended +select count(1) from src where null = 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +select count(1) from src where null = 1 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_FUNCTION + count + 1 + TOK_WHERE + = + TOK_NULL + 1 + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col0 (type: bigint) + auto parallelism: false + Path -> Alias: + -mr-10002default.src{} [src] + Path -> Partition: + -mr-10002default.src{} + Partition + input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.NullStructSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + -mr-10002default.src{} [src] + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(1) from src where null = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from src where null = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 diff --git a/ql/src/test/results/clientpositive/ppd_constant_expr.q.out b/ql/src/test/results/clientpositive/ppd_constant_expr.q.out index 56813e4..17e2bab 100644 --- a/ql/src/test/results/clientpositive/ppd_constant_expr.q.out +++ b/ql/src/test/results/clientpositive/ppd_constant_expr.q.out @@ -32,7 +32,7 @@ STAGE PLANS: alias: src1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: (4.0 + UDFToDouble(null)) (type: double), UDFToInteger((UDFToDouble(key) - UDFToDouble(null))) (type: int), (UDFToDouble(null) + UDFToDouble(null)) (type: double) + expressions: null (type: double), UDFToInteger((UDFToDouble(key) - null)) (type: int), null (type: double) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -168,7 +168,7 @@ STAGE PLANS: alias: src1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: (4.0 + UDFToDouble(null)) (type: double), UDFToInteger((UDFToDouble(key) - UDFToDouble(null))) (type: int), (UDFToDouble(null) + UDFToDouble(null)) (type: double) + expressions: null (type: double), UDFToInteger((UDFToDouble(key) - null)) (type: int), null (type: double) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/auto_join8.q.out b/ql/src/test/results/clientpositive/spark/auto_join8.q.out index 5b6cc80..e0df9a2 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join8.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join8.q.out @@ -96,7 +96,7 @@ STAGE PLANS: predicate: _col2 is null (type: boolean) Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), UDFToInteger(null) (type: int), _col3 (type: string) + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), null (type: int), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/join8.q.out b/ql/src/test/results/clientpositive/spark/join8.q.out index dcfbc3d..d242c10 100644 --- a/ql/src/test/results/clientpositive/spark/join8.q.out +++ b/ql/src/test/results/clientpositive/spark/join8.q.out @@ -102,7 +102,7 @@ STAGE PLANS: predicate: _col2 is null (type: boolean) Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), UDFToInteger(null) (type: int), _col3 (type: string) + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), null (type: int), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out b/ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out index 66db7bd..ca36ac6 100644 --- a/ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out +++ b/ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out @@ -73,15 +73,15 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'k1' (type: string), UDFToString(null) (type: string) + expressions: 'k1' (type: string), null (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 85000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 2 - Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string) Map 3 Map Operator Tree: diff --git a/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out b/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out index 881f41a..02d9b6e 100644 --- a/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out +++ b/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out @@ -1932,3 +1932,156 @@ POSTHOOK: query: select * from src s1, src s2 where false and s1.value = s2.valu POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +PREHOOK: query: explain extended +select count(1) from src where null = 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +select count(1) from src where null = 1 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_FUNCTION + count + 1 + TOK_WHERE + = + TOK_NULL + 1 + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col0 (type: bigint) + auto parallelism: false + Path -> Alias: + -mr-10003default.src{} [src] + Path -> Partition: + -mr-10003default.src{} + Partition + input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.NullStructSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + -mr-10003default.src{} [src] + Reducer 2 + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(1) from src where null = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from src where null = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 diff --git a/ql/src/test/results/clientpositive/spark/vector_elt.q.out b/ql/src/test/results/clientpositive/spark/vector_elt.q.out index 180ea15..818e266 100644 --- a/ql/src/test/results/clientpositive/spark/vector_elt.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_elt.q.out @@ -89,7 +89,7 @@ STAGE PLANS: alias: alltypesorc Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'defg' (type: string), 'cc' (type: string), 'abc' (type: string), '2' (type: string), '12345' (type: string), '123456789012' (type: string), '1.25' (type: string), '16.0' (type: string), null (type: void), null (type: void) + expressions: 'defg' (type: string), 'cc' (type: string), 'abc' (type: string), '2' (type: string), '12345' (type: string), '123456789012' (type: string), '1.25' (type: string), '16.0' (type: string), null (type: string), null (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 12288 Data size: 8687616 Basic stats: COMPLETE Column stats: COMPLETE Limit diff --git a/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out b/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out index 104654a..117dbd0 100644 --- a/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out +++ b/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out @@ -1941,3 +1941,156 @@ POSTHOOK: query: select * from src s1, src s2 where false and s1.value = s2.valu POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +PREHOOK: query: explain extended +select count(1) from src where null = 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +select count(1) from src where null = 1 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_FUNCTION + count + 1 + TOK_WHERE + = + TOK_NULL + 1 + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col0 (type: bigint) + auto parallelism: false + Path -> Alias: + -mr-10002default.src{} [src] + Path -> Partition: + -mr-10002default.src{} + Partition + input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.NullStructSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + -mr-10002default.src{} [src] + Reducer 2 + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(1) from src where null = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from src where null = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 diff --git a/ql/src/test/results/clientpositive/tez/vector_coalesce.q.out b/ql/src/test/results/clientpositive/tez/vector_coalesce.q.out index 2f7eb43..db5e4f8 100644 --- a/ql/src/test/results/clientpositive/tez/vector_coalesce.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_coalesce.q.out @@ -68,7 +68,7 @@ STAGE PLANS: Filter Operator predicate: ctinyint is null (type: boolean) Select Operator - expressions: null (type: void), cdouble (type: double), cint (type: int), COALESCE((UDFToInteger(null) + 10),(cdouble + log2(cint)),0) (type: double) + expressions: null (type: void), cdouble (type: double), cint (type: int), COALESCE(null,(cdouble + log2(cint)),0) (type: double) outputColumnNames: _col0, _col1, _col2, _col3 Limit Number of rows: 10 @@ -117,7 +117,7 @@ STAGE PLANS: Filter Operator predicate: (cfloat is null and cbigint is null) (type: boolean) Select Operator - expressions: null (type: void), null (type: void), COALESCE(null,null,0) (type: float) + expressions: null (type: void), null (type: void), 0 (type: int) outputColumnNames: _col0, _col1, _col2 Limit Number of rows: 10 diff --git a/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out b/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out index 96f19ac..b24ebcc 100644 --- a/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out @@ -1191,7 +1191,7 @@ STAGE PLANS: alias: decimal_udf Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: (UDFToDouble(key) / UDFToDouble(null)) (type: double) + expressions: (UDFToDouble(key) / null) (type: double) outputColumnNames: _col0 Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE Limit @@ -1204,7 +1204,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/tez/vector_elt.q.out b/ql/src/test/results/clientpositive/tez/vector_elt.q.out index b27798a..bb66867 100644 --- a/ql/src/test/results/clientpositive/tez/vector_elt.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_elt.q.out @@ -84,7 +84,7 @@ STAGE PLANS: TableScan alias: alltypesorc Select Operator - expressions: 'defg' (type: string), 'cc' (type: string), 'abc' (type: string), '2' (type: string), '12345' (type: string), '123456789012' (type: string), '1.25' (type: string), '16.0' (type: string), null (type: void), null (type: void) + expressions: 'defg' (type: string), 'cc' (type: string), 'abc' (type: string), '2' (type: string), '12345' (type: string), '123456789012' (type: string), '1.25' (type: string), '16.0' (type: string), null (type: string), null (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Limit Number of rows: 1 diff --git a/ql/src/test/results/clientpositive/udf4.q.out b/ql/src/test/results/clientpositive/udf4.q.out index 1dfd7f8..1e2ba3f 100644 --- a/ql/src/test/results/clientpositive/udf4.q.out +++ b/ql/src/test/results/clientpositive/udf4.q.out @@ -77,7 +77,7 @@ STAGE PLANS: alias: dest1 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 1.0 (type: double), 2.0 (type: double), -2.0 (type: double), 1 (type: bigint), 1 (type: bigint), -2 (type: bigint), 1.0 (type: double), null (type: void), 0.0 (type: double), 1 (type: bigint), 2 (type: bigint), -1 (type: bigint), 1 (type: bigint), rand(3) (type: double), 3 (type: int), -3 (type: int), 3 (type: int), -1 (type: int), -2 (type: int), -2 (type: tinyint), -2 (type: smallint), -2 (type: bigint), 0 (type: tinyint), 0 (type: smallint), 0 (type: int), 0 (type: bigint), 3 (type: tinyint), 3 (type: smallint), 3 (type: int), 3 (type: bigint), 2 (type: tinyint), 2 (type: smallint), 2 (type: int), 2 (type: bigint) + expressions: 1.0 (type: double), 2.0 (type: double), -2.0 (type: double), 1 (type: bigint), 1 (type: bigint), -2 (type: bigint), 1.0 (type: double), null (type: double), 0.0 (type: double), 1 (type: bigint), 2 (type: bigint), -1 (type: bigint), 1 (type: bigint), rand(3) (type: double), 3 (type: int), -3 (type: int), 3 (type: int), -1 (type: int), -2 (type: int), -2 (type: tinyint), -2 (type: smallint), -2 (type: bigint), 0 (type: tinyint), 0 (type: smallint), 0 (type: int), 0 (type: bigint), 3 (type: tinyint), 3 (type: smallint), 3 (type: int), 3 (type: bigint), 2 (type: tinyint), 2 (type: smallint), 2 (type: int), 2 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33 Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git a/ql/src/test/results/clientpositive/udf6.q.out b/ql/src/test/results/clientpositive/udf6.q.out index 1de47ab..0b92bd5 100644 --- a/ql/src/test/results/clientpositive/udf6.q.out +++ b/ql/src/test/results/clientpositive/udf6.q.out @@ -64,33 +64,21 @@ SELECT IF(TRUE, 1, 2), IF(FALSE, 1, 2), IF(NULL, 1, 2), IF(TRUE, "a", "b"), CAST('128' AS STRING) FROM dest1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: dest1 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 1 (type: int), 2 (type: int), if(null, 1, 2) (type: int), 'a' (type: string), 0.1 (type: double), 2 (type: bigint), 126 (type: tinyint), 128 (type: smallint), 128 (type: int), 1.0 (type: double), '128' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: dest1 + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int), 2 (type: int), 2 (type: int), 'a' (type: string), 0.1 (type: double), 2 (type: bigint), 126 (type: tinyint), 128 (type: smallint), 128 (type: int), 1.0 (type: double), '128' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: COMPLETE + ListSink PREHOOK: query: SELECT IF(TRUE, 1, 2), IF(FALSE, 1, 2), IF(NULL, 1, 2), IF(TRUE, "a", "b"), IF(TRUE, 0.1, 0.2), IF(FALSE, CAST(1 AS BIGINT), CAST(2 AS BIGINT)), diff --git a/ql/src/test/results/clientpositive/udf7.q.out b/ql/src/test/results/clientpositive/udf7.q.out index e616fed..b15ccaf 100644 --- a/ql/src/test/results/clientpositive/udf7.q.out +++ b/ql/src/test/results/clientpositive/udf7.q.out @@ -47,7 +47,7 @@ STAGE PLANS: alias: dest1 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 1.098612288668 (type: double), null (type: void), null (type: void), 1.098612288668 (type: double), null (type: void), null (type: void), 1.584962500721 (type: double), null (type: void), null (type: void), 0.47712125472 (type: double), null (type: void), null (type: void), 1.584962500721 (type: double), null (type: void), null (type: void), null (type: void), -1.0 (type: double), 7.389056098931 (type: double), 8.0 (type: double), 8.0 (type: double), 0.125 (type: double), 8.0 (type: double), 2.0 (type: double), NaN (type: double), 1.0 (type: double), 1.0 (type: double), 8.0 (type: double), 8.0 (type: double) + expressions: 1.098612288668 (type: double), null (type: double), null (type: double), 1.098612288668 (type: double), null (type: double), null (type: double), 1.584962500721 (type: double), null (type: double), null (type: double), 0.47712125472 (type: double), null (type: double), null (type: double), 1.584962500721 (type: double), null (type: double), null (type: double), null (type: double), -1.0 (type: double), 7.389056098931 (type: double), 8.0 (type: double), 8.0 (type: double), 0.125 (type: double), 8.0 (type: double), 2.0 (type: double), NaN (type: double), 1.0 (type: double), 1.0 (type: double), 8.0 (type: double), 8.0 (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27 Statistics: Num rows: 1 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE ListSink diff --git a/ql/src/test/results/clientpositive/udf_case.q.out b/ql/src/test/results/clientpositive/udf_case.q.out index ed0aac0..05c65c9 100644 --- a/ql/src/test/results/clientpositive/udf_case.q.out +++ b/ql/src/test/results/clientpositive/udf_case.q.out @@ -89,7 +89,7 @@ STAGE PLANS: Row Limit Per Split: 1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 2 (type: int), 5 (type: int), 15 (type: int), null (type: void), CASE (17) WHEN (18) THEN (null) WHEN (17) THEN (20) END (type: int), 24 (type: int) + expressions: 2 (type: int), 5 (type: int), 15 (type: int), null (type: int), 20 (type: int), 24 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 500 Data size: 10000 Basic stats: COMPLETE Column stats: COMPLETE ListSink diff --git a/ql/src/test/results/clientpositive/udf_coalesce.q.out b/ql/src/test/results/clientpositive/udf_coalesce.q.out index 322dc4e..83024f2 100644 --- a/ql/src/test/results/clientpositive/udf_coalesce.q.out +++ b/ql/src/test/results/clientpositive/udf_coalesce.q.out @@ -66,9 +66,9 @@ STAGE PLANS: Row Limit Per Split: 1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 1 (type: int), 1 (type: int), COALESCE(null,2) (type: int), COALESCE(1,null) (type: int), COALESCE(null,null,3) (type: int), COALESCE(4,null,null,null) (type: int), '1' (type: string), '1' (type: string), COALESCE(null,'2') (type: string), COALESCE('1',null) (type: string), COALESCE(null,null,'3') (type: string), COALESCE('4',null,null,null) (type: string), 1.0 (type: double), 1.0 (type: double), COALESCE(null,2.0) (type: double), COALESCE(null,2.0,3.0) (type: double), COALESCE(2.0,null,3.0) (type: double), COALESCE(if(true, null, 0),null) (type: int) + expressions: 1 (type: int), 1 (type: int), 2 (type: int), 1 (type: int), 3 (type: int), 4 (type: int), '1' (type: string), '1' (type: string), '2' (type: string), '1' (type: string), '3' (type: string), '4' (type: string), 1.0 (type: double), 1.0 (type: double), 2.0 (type: double), 2.0 (type: double), 2.0 (type: double), null (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 500 Data size: 289000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 287000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT COALESCE(1), diff --git a/ql/src/test/results/clientpositive/udf_elt.q.out b/ql/src/test/results/clientpositive/udf_elt.q.out index f8acbf2..18f3481 100644 --- a/ql/src/test/results/clientpositive/udf_elt.q.out +++ b/ql/src/test/results/clientpositive/udf_elt.q.out @@ -52,9 +52,9 @@ STAGE PLANS: Row Limit Per Split: 1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'defg' (type: string), 'cc' (type: string), 'abc' (type: string), '2' (type: string), '12345' (type: string), '123456789012' (type: string), '1.25' (type: string), '16.0' (type: string), elt(null, 'abc', 'defg') (type: string), null (type: void), null (type: void) + expressions: 'defg' (type: string), 'cc' (type: string), 'abc' (type: string), '2' (type: string), '12345' (type: string), '123456789012' (type: string), '1.25' (type: string), '16.0' (type: string), null (type: string), null (type: string), null (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 500 Data size: 395500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 353500 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT elt(2, 'abc', 'defg'), diff --git a/ql/src/test/results/clientpositive/udf_greatest.q.out b/ql/src/test/results/clientpositive/udf_greatest.q.out index 884095b..10f1c2d 100644 --- a/ql/src/test/results/clientpositive/udf_greatest.q.out +++ b/ql/src/test/results/clientpositive/udf_greatest.q.out @@ -58,9 +58,9 @@ STAGE PLANS: Row Limit Per Split: 1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'c' (type: string), 'a' (type: string), 'AaA' (type: string), 'AAA' (type: string), '13' (type: string), '2' (type: string), '03' (type: string), '1' (type: string), greatest(null,'b','c') (type: string), greatest('a',null,'c') (type: string), greatest('a','b',null) (type: string), greatest('a',null,null) (type: string), greatest(null,'b',null) (type: string), greatest(UDFToString(null),null,null) (type: string) + expressions: 'c' (type: string), 'a' (type: string), 'AaA' (type: string), 'AAA' (type: string), '13' (type: string), '2' (type: string), '03' (type: string), '1' (type: string), 'c' (type: string), 'c' (type: string), 'b' (type: string), 'a' (type: string), 'b' (type: string), null (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 500 Data size: 597500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 555500 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT GREATEST('a', 'b', 'c'), diff --git a/ql/src/test/results/clientpositive/udf_hour.q.out b/ql/src/test/results/clientpositive/udf_hour.q.out index 4eb5a00..cf7f78b 100644 --- a/ql/src/test/results/clientpositive/udf_hour.q.out +++ b/ql/src/test/results/clientpositive/udf_hour.q.out @@ -39,7 +39,7 @@ STAGE PLANS: predicate: (UDFToDouble(key) = 86.0) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 13 (type: int), 13 (type: int), null (type: void) + expressions: 13 (type: int), 13 (type: int), null (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE ListSink diff --git a/ql/src/test/results/clientpositive/udf_if.q.out b/ql/src/test/results/clientpositive/udf_if.q.out index a2d2c08..fe41924 100644 --- a/ql/src/test/results/clientpositive/udf_if.q.out +++ b/ql/src/test/results/clientpositive/udf_if.q.out @@ -39,9 +39,9 @@ STAGE PLANS: Row Limit Per Split: 1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 1 (type: int), if(false, UDFToString(null), '1') (type: string), 1 (type: int), if(true, 1, null) (type: int), if(true, null, 1) (type: int), if(if(true, null, false), 1, 2) (type: int) + expressions: 1 (type: int), '1' (type: string), 1 (type: int), 1 (type: int), null (type: int), 2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 500 Data size: 52500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 50500 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT IF(TRUE, 1, 2) AS COL1, diff --git a/ql/src/test/results/clientpositive/udf_instr.q.out b/ql/src/test/results/clientpositive/udf_instr.q.out index 812f244..eafd8d6 100644 --- a/ql/src/test/results/clientpositive/udf_instr.q.out +++ b/ql/src/test/results/clientpositive/udf_instr.q.out @@ -56,9 +56,9 @@ STAGE PLANS: Row Limit Per Split: 1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 1 (type: int), 0 (type: int), 2 (type: int), 2 (type: int), 0 (type: int), 0 (type: int), 2 (type: int), 3 (type: int), 4 (type: int), 2 (type: int), 3 (type: int), instr(null, 'abc') (type: int), instr('abcd', null) (type: int) + expressions: 1 (type: int), 0 (type: int), 2 (type: int), 2 (type: int), 0 (type: int), 0 (type: int), 2 (type: int), 3 (type: int), 4 (type: int), 2 (type: int), 3 (type: int), null (type: int), null (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 500 Data size: 26000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 22000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT instr('abcd', 'abc'), diff --git a/ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out b/ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out index a7d45ea..d459ce3 100644 --- a/ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out +++ b/ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out @@ -44,7 +44,7 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: null is null (type: boolean), true (type: boolean), true (type: boolean) + expressions: true (type: boolean), true (type: boolean), true (type: boolean) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 6000 Basic stats: COMPLETE Column stats: COMPLETE Limit diff --git a/ql/src/test/results/clientpositive/udf_least.q.out b/ql/src/test/results/clientpositive/udf_least.q.out index 95e3467..6983137 100644 --- a/ql/src/test/results/clientpositive/udf_least.q.out +++ b/ql/src/test/results/clientpositive/udf_least.q.out @@ -58,9 +58,9 @@ STAGE PLANS: Row Limit Per Split: 1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'a' (type: string), 'B' (type: string), 'AAA' (type: string), 'A' (type: string), '11' (type: string), '11' (type: string), '01' (type: string), '01' (type: string), least(null,'b','c') (type: string), least('a',null,'c') (type: string), least('a','b',null) (type: string), least('a',null,null) (type: string), least(null,'b',null) (type: string), least(UDFToString(null),null,null) (type: string) + expressions: 'a' (type: string), 'B' (type: string), 'AAA' (type: string), 'A' (type: string), '11' (type: string), '11' (type: string), '01' (type: string), '01' (type: string), 'b' (type: string), 'a' (type: string), 'a' (type: string), 'a' (type: string), 'b' (type: string), null (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 500 Data size: 597500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 555500 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT LEAST('a', 'b', 'c'), diff --git a/ql/src/test/results/clientpositive/udf_locate.q.out b/ql/src/test/results/clientpositive/udf_locate.q.out index 1d10ecd..1bf2b3a 100644 --- a/ql/src/test/results/clientpositive/udf_locate.q.out +++ b/ql/src/test/results/clientpositive/udf_locate.q.out @@ -64,9 +64,9 @@ STAGE PLANS: Row Limit Per Split: 1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 1 (type: int), 0 (type: int), 2 (type: int), 2 (type: int), 4 (type: int), 4 (type: int), 0 (type: int), 0 (type: int), 2 (type: int), 3 (type: int), 4 (type: int), 2 (type: int), 3 (type: int), locate(null, 'abc') (type: int), locate('abc', null) (type: int), locate('abc', 'abcd', null) (type: int), 0 (type: int) + expressions: 1 (type: int), 0 (type: int), 2 (type: int), 2 (type: int), 4 (type: int), 4 (type: int), 0 (type: int), 0 (type: int), 2 (type: int), 3 (type: int), 4 (type: int), 2 (type: int), 3 (type: int), null (type: int), null (type: int), 0 (type: int), 0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 500 Data size: 34000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 30000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT locate('abc', 'abcd'), diff --git a/ql/src/test/results/clientpositive/udf_minute.q.out b/ql/src/test/results/clientpositive/udf_minute.q.out index ebd07c5..3af42c4 100644 --- a/ql/src/test/results/clientpositive/udf_minute.q.out +++ b/ql/src/test/results/clientpositive/udf_minute.q.out @@ -39,7 +39,7 @@ STAGE PLANS: predicate: (UDFToDouble(key) = 86.0) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 14 (type: int), 14 (type: int), null (type: void) + expressions: 14 (type: int), 14 (type: int), null (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/udf_nvl.q.out b/ql/src/test/results/clientpositive/udf_nvl.q.out index 5042577..50ca083 100644 --- a/ql/src/test/results/clientpositive/udf_nvl.q.out +++ b/ql/src/test/results/clientpositive/udf_nvl.q.out @@ -34,7 +34,7 @@ STAGE PLANS: Row Limit Per Split: 1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 1 (type: int), if null is null returns5 (type: int) + expressions: 1 (type: int), 5 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE ListSink diff --git a/ql/src/test/results/clientpositive/udf_parse_url.q.out b/ql/src/test/results/clientpositive/udf_parse_url.q.out index f657fa9..5ea8f40 100644 --- a/ql/src/test/results/clientpositive/udf_parse_url.q.out +++ b/ql/src/test/results/clientpositive/udf_parse_url.q.out @@ -60,7 +60,7 @@ STAGE PLANS: predicate: (UDFToDouble(key) = 86.0) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'facebook.com' (type: string), '/path1/p.php' (type: string), 'k1=v1&k2=v2' (type: string), 'Ref1' (type: string), 'v2' (type: string), 'v1' (type: string), null (type: void), '/path1/p.php?k1=v1&k2=v2' (type: string), 'http' (type: string), null (type: void), 'facebook.com' (type: string) + expressions: 'facebook.com' (type: string), '/path1/p.php' (type: string), 'k1=v1&k2=v2' (type: string), 'Ref1' (type: string), 'v2' (type: string), 'v1' (type: string), null (type: string), '/path1/p.php?k1=v1&k2=v2' (type: string), 'http' (type: string), null (type: string), 'facebook.com' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/udf_second.q.out b/ql/src/test/results/clientpositive/udf_second.q.out index fcd1143..4ee919a 100644 --- a/ql/src/test/results/clientpositive/udf_second.q.out +++ b/ql/src/test/results/clientpositive/udf_second.q.out @@ -39,7 +39,7 @@ STAGE PLANS: predicate: (UDFToDouble(key) = 86.0) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 15 (type: int), 15 (type: int), null (type: void) + expressions: 15 (type: int), 15 (type: int), null (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE ListSink diff --git a/ql/src/test/results/clientpositive/udf_size.q.out b/ql/src/test/results/clientpositive/udf_size.q.out index 95b8e61..efb06a1 100644 --- a/ql/src/test/results/clientpositive/udf_size.q.out +++ b/ql/src/test/results/clientpositive/udf_size.q.out @@ -41,7 +41,7 @@ STAGE PLANS: predicate: (lint is not null and (not mstringstring is null)) (type: boolean) Statistics: Num rows: 3 Data size: 837 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: size(lint) (type: int), size(lintstring) (type: int), size(mstringstring) (type: int), size(null) (type: int) + expressions: size(lint) (type: int), size(lintstring) (type: int), size(mstringstring) (type: int), -1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 837 Basic stats: COMPLETE Column stats: NONE Limit diff --git a/ql/src/test/results/clientpositive/udf_trunc.q.out b/ql/src/test/results/clientpositive/udf_trunc.q.out index b9b2c48..6cc18c2 100644 --- a/ql/src/test/results/clientpositive/udf_trunc.q.out +++ b/ql/src/test/results/clientpositive/udf_trunc.q.out @@ -567,34 +567,22 @@ SELECT TRUNC('2014-01-01', CAST(null AS STRING)) POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: _dummy_table - Row Limit Per Split: 1 - Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - expressions: '2014-02-01' (type: string), trunc(UDFToString(null), 'MM') (type: string), trunc(CAST( null AS DATE), 'MM') (type: string), trunc(CAST( null AS TIMESTAMP), 'MM') (type: string), null (type: void), trunc('2014-01-01', UDFToString(null)) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: '2014-02-01' (type: string), null (type: string), null (type: string), null (type: string), null (type: string), null (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink PREHOOK: query: SELECT TRUNC('2014-01-34', 'MM'), @@ -638,34 +626,22 @@ SELECT TRUNC('2014-01-01', CAST(null AS STRING)) POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: _dummy_table - Row Limit Per Split: 1 - Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - expressions: '2014-01-01' (type: string), trunc(UDFToString(null), 'YEAR') (type: string), trunc(CAST( null AS DATE), 'YEAR') (type: string), trunc(CAST( null AS TIMESTAMP), 'YEAR') (type: string), null (type: void), trunc('2014-01-01', UDFToString(null)) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: '2014-01-01' (type: string), null (type: string), null (type: string), null (type: string), null (type: string), null (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink PREHOOK: query: SELECT TRUNC('2014-01-34', 'YEAR'), diff --git a/ql/src/test/results/clientpositive/udf_when.q.out b/ql/src/test/results/clientpositive/udf_when.q.out index 52f15b3..3681908 100644 --- a/ql/src/test/results/clientpositive/udf_when.q.out +++ b/ql/src/test/results/clientpositive/udf_when.q.out @@ -89,9 +89,9 @@ STAGE PLANS: Row Limit Per Split: 1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 2 (type: int), 9 (type: int), 14 (type: int), null (type: void), CASE WHEN (false) THEN (null) WHEN (true) THEN (24) END (type: int), CASE WHEN (false) THEN (27) WHEN (true) THEN (null) END (type: int) + expressions: 2 (type: int), 9 (type: int), 14 (type: int), null (type: int), 24 (type: int), null (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 500 Data size: 10000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT CASE diff --git a/ql/src/test/results/clientpositive/vector_coalesce.q.out b/ql/src/test/results/clientpositive/vector_coalesce.q.out index 096ee22..c63f2d1 100644 --- a/ql/src/test/results/clientpositive/vector_coalesce.q.out +++ b/ql/src/test/results/clientpositive/vector_coalesce.q.out @@ -87,7 +87,7 @@ STAGE PLANS: predicate: ctinyint is null (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: null (type: void), cdouble (type: double), cint (type: int), COALESCE((UDFToInteger(null) + 10),(cdouble + log2(cint)),0) (type: double) + expressions: null (type: void), cdouble (type: double), cint (type: int), COALESCE(null,(cdouble + log2(cint)),0) (type: double) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Limit @@ -153,7 +153,7 @@ STAGE PLANS: predicate: (cfloat is null and cbigint is null) (type: boolean) Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: null (type: void), null (type: void), COALESCE(null,null,0) (type: float) + expressions: null (type: void), null (type: void), 0 (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE Limit diff --git a/ql/src/test/results/clientpositive/vector_decimal_udf.q.out b/ql/src/test/results/clientpositive/vector_decimal_udf.q.out index 189e85f..e381757 100644 --- a/ql/src/test/results/clientpositive/vector_decimal_udf.q.out +++ b/ql/src/test/results/clientpositive/vector_decimal_udf.q.out @@ -1146,7 +1146,7 @@ STAGE PLANS: alias: decimal_udf Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: (UDFToDouble(key) / UDFToDouble(null)) (type: double) + expressions: (UDFToDouble(key) / null) (type: double) outputColumnNames: _col0 Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE Limit @@ -1159,7 +1159,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/vector_elt.q.out b/ql/src/test/results/clientpositive/vector_elt.q.out index 3a2c1fc..e3fa9ed 100644 --- a/ql/src/test/results/clientpositive/vector_elt.q.out +++ b/ql/src/test/results/clientpositive/vector_elt.q.out @@ -102,7 +102,7 @@ STAGE PLANS: alias: alltypesorc Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'defg' (type: string), 'cc' (type: string), 'abc' (type: string), '2' (type: string), '12345' (type: string), '123456789012' (type: string), '1.25' (type: string), '16.0' (type: string), null (type: void), null (type: void) + expressions: 'defg' (type: string), 'cc' (type: string), 'abc' (type: string), '2' (type: string), '12345' (type: string), '123456789012' (type: string), '1.25' (type: string), '16.0' (type: string), null (type: string), null (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 12288 Data size: 8687616 Basic stats: COMPLETE Column stats: COMPLETE Limit