diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index fb88c12726..4e0527437a 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -260,10 +260,12 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\ vector_count_distinct.q,\ vector_data_types.q,\ vector_date_1.q,\ + vector_decimal64_add_decimal64column_diffscale.q,\ vector_decimal64_div_decimal64scalar.q,\ vector_decimal64_div_decimal64column.q,\ vector_decimal64_mul_decimal64scalar.q,\ vector_decimal64_mul_decimal64column.q,\ + vector_decimal64_mul_intcolumn.q,\ vector_decimal_1.q,\ vector_decimal_10_0.q,\ vector_decimal_2.q,\ diff --git a/ql/src/gen/vectorization/ExpressionTemplates/Decimal64ColumnScaleUp.txt b/ql/src/gen/vectorization/ExpressionTemplates/Decimal64ColumnScaleUp.txt new file mode 100644 index 0000000000..3e84ecdcb1 --- /dev/null +++ b/ql/src/gen/vectorization/ExpressionTemplates/Decimal64ColumnScaleUp.txt @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; + +/** + * Generated from template Decimal64ColumnArithmeticDecimal64ScalarUnscaled.txt, which covers decimal64 arithmetic + * expressions between a column and a scalar unscaled version. + */ +public class extends { + + public (int colNum, long value, int outputColumnNum) { + super(colNum, value, outputColumnNum); + } + + public () { + super(); + } + + @Override + public String vectorExpressionParameters() { + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) inputTypeInfos[1]; + HiveDecimalWritable writable = new HiveDecimalWritable(); + writable.deserialize64(value, decimalTypeInfo.scale()); + return getColumnParamString(0, colNum) + ", decimal64Val " + value + + ", decimalVal " + writable.toString(); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.DECIMAL_64, + VectorExpressionDescriptor.ArgumentType.DECIMAL_64) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR) + .setUnscaled(true).build(); + } + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index c8bc4f7b63..979a369012 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -61,6 +61,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.CastFloatToVarChar; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastLongToChar; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastLongToDecimal; +import org.apache.hadoop.hive.ql.exec.vector.expressions.CastLongToDecimal64; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastLongToString; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastLongToTimestamp; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastLongToVarChar; @@ -338,6 +339,7 @@ public VectorizationContext( if (hiveConf != null) { setHiveConfVars(hiveConf); } + this.tryDecimal64Cast = false; } // Convenient constructor for initial batch creation takes @@ -364,6 +366,7 @@ public VectorizationContext(String contextName, List initialColumnNames, if (hiveConf != null) { setHiveConfVars(hiveConf); } + tryDecimal64Cast = false; } public VectorizationContext(String contextName, List initialColumnNames, @@ -387,6 +390,7 @@ public VectorizationContext(String contextName, HiveConf hiveConf) { projectionColumnNames = new ArrayList<>(); projectionColumnMap = new HashMap<>(); this.ocm = new OutputColumnManager(0); + this.tryDecimal64Cast = false; this.firstOutputColumnIndex = 0; vMap = new VectorExpressionDescriptor(); @@ -415,6 +419,7 @@ public VectorizationContext(String contextName, VectorizationContext vContext) { this.projectionColumnMap = new HashMap<>(); this.ocm = vContext.ocm; + this.tryDecimal64Cast = false; this.firstOutputColumnIndex = vContext.firstOutputColumnIndex; vMap = new VectorExpressionDescriptor(); @@ -560,6 +565,9 @@ public DataTypePhysicalVariation getDataTypePhysicalVariation(int columnNum) thr //Map column number to type (this is always non-null for a useful vec context) private OutputColumnManager ocm; + //Can cast to decimal64 + boolean tryDecimal64Cast; + // Set of UDF classes for type casting data types in row-mode. private static final Set> castExpressionUdfs = new HashSet<>(); static { @@ -637,6 +645,28 @@ public DataTypePhysicalVariation getDataTypePhysicalVariation(int columnNum) thr udfsNeedingImplicitDecimalCast.add(UDFOPLongDivide.class); } + private static final long[] POWEROFTENTABLE = { + 1L, // 0 + 10L, + 100L, + 1000L, + 10000L, + 100000L, + 1000000L, + 10000000L, + 100000000L, // 8 + 1000000000L, + 10000000000L, + 100000000000L, + 1000000000000L, + 10000000000000L, + 100000000000000L, + 1000000000000000L, + 10000000000000000L, // 16 + 100000000000000000L, + 1000000000000000000L, // 18 + }; + protected boolean needsImplicitCastForDecimal(GenericUDF udf) { Class udfClass = udf.getClass(); if (udf instanceof GenericUDFBridge) { @@ -1663,7 +1693,9 @@ private boolean checkExprNodeDescForDecimal64(ExprNodeDesc exprNodeDesc) throws GenericUDF udf = ((ExprNodeGenericFuncDesc) exprNodeDesc).getGenericUDF(); Class udfClass = udf.getClass(); - + if (udf instanceof GenericUDFToDecimal) { + return true; + } // We have a class-level annotation that says whether the UDF's vectorization expressions // support Decimal64. VectorizedExpressionsSupportDecimal64 annotation = @@ -1724,6 +1756,7 @@ private VectorExpression getDecimal64VectorExpressionForUdf(GenericUDF genericUd boolean isDecimal64ScaleEstablished = false; int decimal64ColumnScale = 0; boolean hasConstants = false; + boolean scaleMismatch = false; for (int i = 0; i < numChildren; i++) { ExprNodeDesc childExpr = childExprs.get(i); @@ -1742,11 +1775,12 @@ private VectorExpression getDecimal64VectorExpressionForUdf(GenericUDF genericUd childExpr instanceof ExprNodeColumnDesc) { if (isExprDecimal64) { DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo; - if (!isDecimal64ScaleEstablished) { + if (decimalTypeInfo.getScale() != decimal64ColumnScale && i > 0) { + scaleMismatch = true; + } + if (decimalTypeInfo.getScale() >= decimal64ColumnScale) { decimal64ColumnScale = decimalTypeInfo.getScale(); isDecimal64ScaleEstablished = true; - } else if (decimalTypeInfo.getScale() != decimal64ColumnScale) { - return null; } } builder.setInputExpressionType(i, InputExpressionType.COLUMN); @@ -1805,7 +1839,7 @@ private VectorExpression getDecimal64VectorExpressionForUdf(GenericUDF genericUd returnDataTypePhysicalVariation = DataTypePhysicalVariation.NONE; } - if(dontRescaleArguments && hasConstants) { + if (dontRescaleArguments && hasConstants) { builder.setUnscaled(true); } VectorExpressionDescriptor.Descriptor descriptor = builder.build(); @@ -1817,6 +1851,45 @@ private VectorExpression getDecimal64VectorExpressionForUdf(GenericUDF genericUd VectorExpressionDescriptor.Mode childrenMode = getChildrenMode(mode, udfClass); + // Rewrite the operand with smaller scale, so we can process Decimal64 operations + // on all the arithmetic and comparison operations. + if (scaleMismatch && !hasConstants && + (genericUdf instanceof GenericUDFBaseArithmetic + || genericUdf instanceof GenericUDFBaseCompare)) { + ExprNodeDesc left = childExprs.get(0); + ExprNodeDesc right = childExprs.get(1); + DecimalTypeInfo leftTypeInfo = (DecimalTypeInfo)left.getTypeInfo(); + DecimalTypeInfo rightTypeInfo = (DecimalTypeInfo)right.getTypeInfo(); + int leftScale = leftTypeInfo.getScale(); + int rightScale = rightTypeInfo.getScale(); + int leftPrecision = leftTypeInfo.precision(); + int rightPrecision = rightTypeInfo.precision(); + ExprNodeDesc newConstant; + List children = new ArrayList<>(); + DecimalTypeInfo resultTypeInfo; + int childIndexToRewrite = -1; + if (leftScale < rightScale) { + newConstant = new ExprNodeConstantDesc(new DecimalTypeInfo(rightScale - leftScale, 0), + HiveDecimal.create(POWEROFTENTABLE[rightScale - leftScale])); + children.add(childExprs.get(0)); + childIndexToRewrite = 0; + resultTypeInfo = new DecimalTypeInfo(leftPrecision + rightScale - leftScale, + rightScale); + } else { + newConstant = new ExprNodeConstantDesc(new DecimalTypeInfo(leftScale - rightScale, 0), + HiveDecimal.create(POWEROFTENTABLE[leftScale - rightScale])); + childIndexToRewrite = 1; + resultTypeInfo = new DecimalTypeInfo(rightPrecision + leftScale - rightScale, + leftScale); + } + children.add(childExprs.get(childIndexToRewrite)); + children.add(newConstant); + ExprNodeGenericFuncDesc newScaledExpr = new ExprNodeGenericFuncDesc(resultTypeInfo, + new GenericUDFOPScaleUpDecimal64(), " ScaleUp ", children); + childExprs.remove(childIndexToRewrite); + childExprs.add(childIndexToRewrite, newScaledExpr); + } + return createDecimal64VectorExpression( vectorClass, childExprs, childrenMode, isDecimal64ScaleEstablished, decimal64ColumnScale, @@ -1832,7 +1905,7 @@ private VectorExpression createDecimal64VectorExpression(Class vectorClass, throws HiveException { final int numChildren = childExprs.size(); - + tryDecimal64Cast = true; /* * Custom build arguments. */ @@ -1860,6 +1933,7 @@ private VectorExpression createDecimal64VectorExpression(Class vectorClass, VectorExpression filterExpr = getFilterOnBooleanColumnExpression((ExprNodeColumnDesc) childExpr, colIndex); if (filterExpr == null) { + tryDecimal64Cast = false; return null; } @@ -1870,10 +1944,12 @@ private VectorExpression createDecimal64VectorExpression(Class vectorClass, ExprNodeConstantDesc constDesc = (ExprNodeConstantDesc) childExpr; if (typeInfo instanceof DecimalTypeInfo) { if (!isDecimal64ScaleEstablished) { + tryDecimal64Cast = false; return null; } HiveDecimal hiveDecimal = (HiveDecimal) constDesc.getValue(); if (hiveDecimal.scale() > decimal64ColumnScale) { + tryDecimal64Cast = false; // For now, bail out on decimal constants with larger scale than column scale. return null; @@ -1890,6 +1966,7 @@ private VectorExpression createDecimal64VectorExpression(Class vectorClass, getConstantVectorExpression(null, typeInfo, childrenMode) : scalarValue; } } else { + tryDecimal64Cast = false; return null; } } @@ -1911,6 +1988,7 @@ private VectorExpression createDecimal64VectorExpression(Class vectorClass, if (!children.isEmpty()) { vectorExpression.setChildExpressions(children.toArray(new VectorExpression[0])); } + tryDecimal64Cast = false; return vectorExpression; } @@ -1948,7 +2026,7 @@ private VectorExpression getVectorExpressionForUdf(GenericUDF genericUdf, vclass = FilterExprAndExpr.class; } VectorExpressionDescriptor.Mode childrenMode = getChildrenMode(mode, udfClass); - return createVectorExpression(vclass, childExpr, childrenMode, returnType); + return createVectorExpression(vclass, childExpr, childrenMode, returnType, DataTypePhysicalVariation.NONE); } if (numChildren > VectorExpressionDescriptor.MAX_NUM_ARGUMENTS) { return null; @@ -2003,7 +2081,7 @@ private VectorExpression getVectorExpressionForUdf(GenericUDF genericUdf, return null; } VectorExpressionDescriptor.Mode childrenMode = getChildrenMode(mode, udfClass); - return createVectorExpression(vclass, childExpr, childrenMode, returnType); + return createVectorExpression(vclass, childExpr, childrenMode, returnType, DataTypePhysicalVariation.NONE); } private VectorExpression createDecimal64ToDecimalConversion(int colIndex, TypeInfo resultTypeInfo) @@ -2065,7 +2143,8 @@ public VectorExpression wrapWithDecimal64ToDecimalConversion(VectorExpression in } private VectorExpression createVectorExpression(Class vectorClass, - List childExpr, VectorExpressionDescriptor.Mode childrenMode, TypeInfo returnType) throws HiveException { + List childExpr, VectorExpressionDescriptor.Mode childrenMode, TypeInfo returnType, + DataTypePhysicalVariation returnDataTypePhysicalVariation) throws HiveException { int numChildren = childExpr == null ? 0: childExpr.size(); TypeInfo[] inputTypeInfos = new TypeInfo[numChildren]; @@ -2130,7 +2209,8 @@ private VectorExpression createVectorExpression(Class vectorClass, throw new HiveException("Cannot handle expression type: " + child.getClass().getSimpleName()); } } - VectorExpression vectorExpression = instantiateExpression(vectorClass, returnType, DataTypePhysicalVariation.NONE, arguments); + VectorExpression vectorExpression = instantiateExpression(vectorClass, returnType, returnDataTypePhysicalVariation, + arguments); if (vectorExpression == null) { handleCouldNotInstantiateVectorExpression(vectorClass, returnType, DataTypePhysicalVariation.NONE, arguments); } @@ -2359,7 +2439,7 @@ private VectorExpression getCastToTimestamp(GenericUDFTimestamp udf, // Replace with the milliseconds conversion if (!udf.isIntToTimestampInSeconds() && ve instanceof CastLongToTimestamp) { ve = createVectorExpression(CastMillisecondsLongToTimestamp.class, - childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType, DataTypePhysicalVariation.NONE); } return ve; @@ -2683,7 +2763,8 @@ private VectorExpression getStructInExpression(List childExpr, Exp Class cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterStructColumnInList.class : StructColumnInList.class); - expr = createVectorExpression(cl, null, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + expr = createVectorExpression(cl, null, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); ((IStringInExpr) expr).setInListValues(serializedInChildren); @@ -2742,7 +2823,8 @@ private VectorExpression getInExpression(List childExpr, for (int i = 0; i != inVals.length; i++) { inVals[i] = getIntFamilyScalarAsLong((ExprNodeConstantDesc) childrenForInList.get(i)); } - expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType); + expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); ((ILongInExpr) expr).setInListValues(inVals); } else if (isTimestampFamily(colType)) { cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterTimestampColumnInList.class : TimestampColumnInList.class); @@ -2750,7 +2832,8 @@ private VectorExpression getInExpression(List childExpr, for (int i = 0; i != inVals.length; i++) { inVals[i] = getTimestampScalar(childrenForInList.get(i)); } - expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType); + expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); ((ITimestampInExpr) expr).setInListValues(inVals); } else if (isStringFamily(colType)) { cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterStringColumnInList.class : StringColumnInList.class); @@ -2758,7 +2841,8 @@ private VectorExpression getInExpression(List childExpr, for (int i = 0; i != inVals.length; i++) { inVals[i] = getStringScalarAsByteArray((ExprNodeConstantDesc) childrenForInList.get(i)); } - expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType); + expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); ((IStringInExpr) expr).setInListValues(inVals); } else if (isFloatFamily(colType)) { cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterDoubleColumnInList.class : DoubleColumnInList.class); @@ -2766,7 +2850,8 @@ private VectorExpression getInExpression(List childExpr, for (int i = 0; i != inValsD.length; i++) { inValsD[i] = getNumericScalarAsDouble(childrenForInList.get(i)); } - expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType); + expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); ((IDoubleInExpr) expr).setInListValues(inValsD); } else if (isDecimalFamily(colType)) { @@ -2798,7 +2883,8 @@ private VectorExpression getInExpression(List childExpr, cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterDecimalColumnInList.class : DecimalColumnInList.class); expr = createVectorExpression( - cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType); + cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); HiveDecimal[] inValsD = new HiveDecimal[childrenForInList.size()]; for (int i = 0; i != inValsD.length; i++) { inValsD[i] = (HiveDecimal) getVectorTypeScalarValue( @@ -2812,7 +2898,8 @@ private VectorExpression getInExpression(List childExpr, for (int i = 0; i != inVals.length; i++) { inVals[i] = (Long) getVectorTypeScalarValue((ExprNodeConstantDesc) childrenForInList.get(i)); } - expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType); + expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); ((ILongInExpr) expr).setInListValues(inVals); } @@ -3126,11 +3213,17 @@ private VectorExpression getCastToDecimal(List childExpr, TypeInfo } } if (isIntFamily(inputType)) { - return createVectorExpression(CastLongToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + if (tryDecimal64Cast && ((DecimalTypeInfo)returnType).precision() <= 18) + return createVectorExpression(CastLongToDecimal64.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.DECIMAL_64); + return createVectorExpression(CastLongToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (inputType.equals("float")) { - return createVectorExpression(CastFloatToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastFloatToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (inputType.equals("double")) { - return createVectorExpression(CastDoubleToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastDoubleToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (decimalTypePattern.matcher(inputType).matches()) { if (child instanceof ExprNodeColumnDesc) { int colIndex = getInputColumnIndex((ExprNodeColumnDesc) child); @@ -3140,17 +3233,19 @@ private VectorExpression getCastToDecimal(List childExpr, TypeInfo // Do Decimal64 conversion instead. return createDecimal64ToDecimalConversion(colIndex, returnType); } else { - return createVectorExpression(CastDecimalToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, - returnType); + return createVectorExpression(CastDecimalToDecimal.class, childExpr, + VectorExpressionDescriptor.Mode.PROJECTION, returnType, DataTypePhysicalVariation.NONE); } } else { return createVectorExpression(CastDecimalToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, - returnType); + returnType, DataTypePhysicalVariation.NONE); } } else if (isStringFamily(inputType)) { - return createVectorExpression(CastStringToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastStringToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (inputType.equals("timestamp")) { - return createVectorExpression(CastTimestampToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastTimestampToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } return null; } @@ -3174,19 +3269,26 @@ private VectorExpression getCastToString(List childExpr, TypeInfo } if (inputType.equals("boolean")) { // Boolean must come before the integer family. It's a special case. - return createVectorExpression(CastBooleanToStringViaLongToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastBooleanToStringViaLongToString.class, childExpr, + VectorExpressionDescriptor.Mode.PROJECTION, returnType, DataTypePhysicalVariation.NONE); } else if (isIntFamily(inputType)) { - return createVectorExpression(CastLongToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastLongToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (inputType.equals("float")) { - return createVectorExpression(CastFloatToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastFloatToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (inputType.equals("double")) { - return createVectorExpression(CastDoubleToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastDoubleToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (isDecimalFamily(inputType)) { - return createVectorExpression(CastDecimalToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastDecimalToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (isDateFamily(inputType)) { - return createVectorExpression(CastDateToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastDateToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (isTimestampFamily(inputType)) { - return createVectorExpression(CastTimestampToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastTimestampToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (isStringFamily(inputType)) { // STRING and VARCHAR types require no conversion, so use a no-op. @@ -3208,21 +3310,29 @@ private VectorExpression getCastToChar(List childExpr, TypeInfo re } if (inputType.equals("boolean")) { // Boolean must come before the integer family. It's a special case. - return createVectorExpression(CastBooleanToCharViaLongToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastBooleanToCharViaLongToChar.class, childExpr, + VectorExpressionDescriptor.Mode.PROJECTION, returnType, DataTypePhysicalVariation.NONE); } else if (isIntFamily(inputType)) { - return createVectorExpression(CastLongToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastLongToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (inputType.equals("float")) { - return createVectorExpression(CastFloatToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastFloatToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (inputType.equals("double")) { - return createVectorExpression(CastDoubleToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastDoubleToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (isDecimalFamily(inputType)) { - return createVectorExpression(CastDecimalToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastDecimalToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (isDateFamily(inputType)) { - return createVectorExpression(CastDateToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastDateToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (isTimestampFamily(inputType)) { - return createVectorExpression(CastTimestampToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastTimestampToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (isStringFamily(inputType)) { - return createVectorExpression(CastStringGroupToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastStringGroupToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } return null; } @@ -3238,21 +3348,29 @@ private VectorExpression getCastToVarChar(List childExpr, TypeInfo } if (inputType.equals("boolean")) { // Boolean must come before the integer family. It's a special case. - return createVectorExpression(CastBooleanToVarCharViaLongToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastBooleanToVarCharViaLongToVarChar.class, childExpr, + VectorExpressionDescriptor.Mode.PROJECTION, returnType, DataTypePhysicalVariation.NONE); } else if (isIntFamily(inputType)) { - return createVectorExpression(CastLongToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastLongToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (inputType.equals("float")) { - return createVectorExpression(CastFloatToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastFloatToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (inputType.equals("double")) { - return createVectorExpression(CastDoubleToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastDoubleToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (isDecimalFamily(inputType)) { - return createVectorExpression(CastDecimalToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastDecimalToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (isDateFamily(inputType)) { - return createVectorExpression(CastDateToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastDateToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (isTimestampFamily(inputType)) { - return createVectorExpression(CastTimestampToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastTimestampToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } else if (isStringFamily(inputType)) { - return createVectorExpression(CastStringGroupToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastStringGroupToVarChar.class, childExpr, + VectorExpressionDescriptor.Mode.PROJECTION, returnType, DataTypePhysicalVariation.NONE); } return null; } @@ -3271,7 +3389,8 @@ private VectorExpression getCastToBinary(List childExpr, TypeInfo // STRING and VARCHAR types require no conversion, so use a no-op. return getIdentityExpression(childExpr); } else if (charTypePattern.matcher(inputType).matches()) { - return createVectorExpression(CastCharToBinary.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + return createVectorExpression(CastCharToBinary.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, + returnType, DataTypePhysicalVariation.NONE); } return null; } @@ -3289,13 +3408,15 @@ private VectorExpression getCastToDoubleExpression(Class udf, List childExpr) if (isStringFamily(inputType)) { return createVectorExpression(CastStringToBoolean.class, childExpr, - VectorExpressionDescriptor.Mode.PROJECTION, TypeInfoFactory.booleanTypeInfo); + VectorExpressionDescriptor.Mode.PROJECTION, TypeInfoFactory.booleanTypeInfo, DataTypePhysicalVariation.NONE); } return null; } @@ -3352,7 +3473,7 @@ private VectorExpression getCastWithFormat( Class veClass = getCastFormatVectorExpressionClass(childExpr, returnType, inputType); return createVectorExpression( - veClass, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + veClass, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType, DataTypePhysicalVariation.NONE); } private Class getCastFormatVectorExpressionClass(List childExpr, @@ -3633,7 +3754,7 @@ private VectorExpression getBetweenExpression(List childExpr, } } return createVectorExpression( - cl, childrenAfterNot, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + cl, childrenAfterNot, VectorExpressionDescriptor.Mode.PROJECTION, returnType, DataTypePhysicalVariation.NONE); } private boolean isCondExpr(ExprNodeDesc exprNodeDesc) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastLongToDecimal64.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastLongToDecimal64.java new file mode 100644 index 0000000000..128085594b --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastLongToDecimal64.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; + +/** + * To be used to cast long and boolean to decimal. + * This works for boolean too because boolean is encoded as 0 + * for false and 1 for true. + */ +public class CastLongToDecimal64 extends FuncLongToDecimal64 { + + private static final long serialVersionUID = 1L; + + public CastLongToDecimal64() { + super(); + } + + public CastLongToDecimal64(int inputColumn, int outputColumnNum) { + super(inputColumn, outputColumnNum); + } + + @Override + protected void func(Decimal64ColumnVector outV, LongColumnVector inV, int i) { + outV.vector[i] = inV.vector[i]; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FuncLongToDecimal64.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FuncLongToDecimal64.java new file mode 100644 index 0000000000..4af8fb543c --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FuncLongToDecimal64.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import java.util.Arrays; + +import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressionsSupportDecimal64; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.metadata.HiveException; + +/** + * This is a superclass for unary long functions and expressions returning decimals that + * operate directly on the input and set the output. + */ +@VectorizedExpressionsSupportDecimal64() +public abstract class FuncLongToDecimal64 extends VectorExpression { + private static final long serialVersionUID = 1L; + private final int inputColumn; + + public FuncLongToDecimal64(int inputColumn, int outputColumnNum) { + super(outputColumnNum); + this.inputColumn = inputColumn; + } + + public FuncLongToDecimal64() { + super(); + + // Dummy final assignments. + inputColumn = -1; + } + + abstract protected void func(Decimal64ColumnVector outputColVector, LongColumnVector inputColVector, int i); + + @Override + public void evaluate(VectorizedRowBatch batch) throws HiveException { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + LongColumnVector inputColVector = (LongColumnVector) batch.cols[inputColumn]; + int[] sel = batch.selected; + int n = batch.size; + Decimal64ColumnVector outputColVector = (Decimal64ColumnVector) batch.cols[outputColumnNum]; + + boolean[] inputIsNull = inputColVector.isNull; + boolean[] outputIsNull = outputColVector.isNull; + + if (n == 0) { + + // Nothing to do + return; + } + + // We do not need to do a column reset since we are carefully changing the output. + outputColVector.isRepeating = false; + + if (inputColVector.isRepeating) { + if (inputColVector.noNulls || !inputIsNull[0]) { + // Set isNull before call in case it changes it mind. + outputIsNull[0] = false; + func(outputColVector, inputColVector, 0); + } else { + outputIsNull[0] = true; + outputColVector.noNulls = false; + } + outputColVector.isRepeating = true; + return; + } + + if (inputColVector.noNulls) { + if (batch.selectedInUse) { + + // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop. + + if (!outputColVector.noNulls) { + for(int j = 0; j != n; j++) { + final int i = sel[j]; + // Set isNull before call in case it changes it mind. + outputIsNull[i] = false; + func(outputColVector, inputColVector, i); + } + } else { + for(int j = 0; j != n; j++) { + final int i = sel[j]; + func(outputColVector, inputColVector, i); + } + } + } else { + if (!outputColVector.noNulls) { + + // Assume it is almost always a performance win to fill all of isNull so we can + // safely reset noNulls. + Arrays.fill(outputIsNull, false); + outputColVector.noNulls = true; + } + for(int i = 0; i != n; i++) { + func(outputColVector, inputColVector, i); + } + } + } else /* there are nulls in the inputColVector */ { + + // Carefully handle NULLs... + outputColVector.noNulls = false; + + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputColVector.isNull[i] = inputColVector.isNull[i]; + if (!inputColVector.isNull[i]) { + func(outputColVector, inputColVector, i); + } + } + } else { + System.arraycopy(inputColVector.isNull, 0, outputColVector.isNull, 0, n); + for(int i = 0; i != n; i++) { + if (!inputColVector.isNull[i]) { + func(outputColVector, inputColVector, i); + } + } + } + } + } + + public String vectorExpressionParameters() { + return getColumnParamString(0, inputColumn); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + VectorExpressionDescriptor.Builder b = new VectorExpressionDescriptor.Builder(); + b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(1) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.INT_FAMILY) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN); + return b.build(); + } +} \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 6876787e11..817389aac1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -4708,14 +4708,10 @@ private static VectorExpression fixDecimalDataTypePhysicalVariations(final Vecto if (parent.getOutputDataTypePhysicalVariation() == DataTypePhysicalVariation.NONE) { boolean inputArgsChanged = false; DataTypePhysicalVariation[] dataTypePhysicalVariations = parent.getInputDataTypePhysicalVariations(); - VectorExpression oldExpression = null; - VectorExpression newExpression = null; for (int i = 0; i < children.length; i++) { - oldExpression = children[i]; // we found at least one children with mismatch - if (oldExpression.getOutputDataTypePhysicalVariation() == DataTypePhysicalVariation.DECIMAL_64) { - newExpression = vContext.wrapWithDecimal64ToDecimalConversion(oldExpression); - children[i] = newExpression; + if (children[i].getOutputDataTypePhysicalVariation() == DataTypePhysicalVariation.DECIMAL_64) { + children[i] = vContext.wrapWithDecimal64ToDecimalConversion(children[i]); inputArgsChanged = true; dataTypePhysicalVariations[i] = DataTypePhysicalVariation.NONE; } @@ -4725,9 +4721,9 @@ private static VectorExpression fixDecimalDataTypePhysicalVariations(final Vecto if (parent instanceof VectorUDFAdaptor) { VectorUDFAdaptor parentAdaptor = (VectorUDFAdaptor) parent; VectorUDFArgDesc[] argDescs = parentAdaptor.getArgDescs(); - for (VectorUDFArgDesc argDesc : argDescs) { - if (argDesc.getColumnNum() == oldExpression.getOutputColumnNum()) { - argDesc.setColumnNum(newExpression.getOutputColumnNum()); + for (int i = 0; i < argDescs.length; ++i) { + if (argDescs[i].getColumnNum() != children[i].getOutputColumnNum()) { + argDescs[i].setColumnNum(children[i].getOutputColumnNum()); break; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPScaleUpDecimal64.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPScaleUpDecimal64.java new file mode 100644 index 0000000000..219f29d254 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPScaleUpDecimal64.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressionsSupportDecimal64; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.Decimal64ColScaleUp; + +/* Vectorized Operator to scale up the decimal64 column. + */ +@Description(name = "ScaleUp", value = "a _FUNC_ b - ScalesUp a by multiplying with b") +@VectorizedExpressions({ Decimal64ColScaleUp.class}) +@VectorizedExpressionsSupportDecimal64() +public class GenericUDFOPScaleUpDecimal64 extends GenericUDFOPMultiply { + public GenericUDFOPScaleUpDecimal64() { + super(); + this.opDisplayName = "ScaleUp"; + } +} diff --git a/ql/src/test/queries/clientpositive/vector_decimal64_add_decimal64column_diffscale.q b/ql/src/test/queries/clientpositive/vector_decimal64_add_decimal64column_diffscale.q new file mode 100644 index 0000000000..b98d81f967 --- /dev/null +++ b/ql/src/test/queries/clientpositive/vector_decimal64_add_decimal64column_diffscale.q @@ -0,0 +1,6 @@ +create external table vector_decimal64_mul_decimal64column(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,1), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE; +LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_decimal64column; +create table vector_decimal64_mul_decimal64column_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,1), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) stored as ORC; +insert into table vector_decimal64_mul_decimal64column_tmp select * from vector_decimal64_mul_decimal64column; +explain vectorization detail select sum(ss_ext_list_price+ss_ext_wholesale_cost) from vector_decimal64_mul_decimal64column_tmp; +select sum(ss_ext_list_price+ss_ext_wholesale_cost) from vector_decimal64_mul_decimal64column_tmp; diff --git a/ql/src/test/queries/clientpositive/vector_decimal64_mul_intcolumn.q b/ql/src/test/queries/clientpositive/vector_decimal64_mul_intcolumn.q new file mode 100644 index 0000000000..a4a7c839f1 --- /dev/null +++ b/ql/src/test/queries/clientpositive/vector_decimal64_mul_intcolumn.q @@ -0,0 +1,8 @@ +create external table vector_decimal64_mul_intcolumn(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(19,1), ss_ext_discount_amt int, ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE; +LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_intcolumn; +create table vector_decimal64_mul_intcolumn_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(19,1), ss_ext_discount_amt int, ss_ext_sales_price decimal(7,2)) stored as ORC; +insert into table vector_decimal64_mul_intcolumn_tmp select * from vector_decimal64_mul_intcolumn; +explain vectorization detail select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp; +select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp; +explain vectorization detail select sum(ss_ext_wholesale_cost*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp; +select sum(ss_ext_wholesale_cost*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp; diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal64_add_decimal64column_diffscale.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal64_add_decimal64column_diffscale.q.out new file mode 100644 index 0000000000..cd0be187aa --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/vector_decimal64_add_decimal64column_diffscale.q.out @@ -0,0 +1,171 @@ +PREHOOK: query: create external table vector_decimal64_mul_decimal64column(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,1), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vector_decimal64_mul_decimal64column +POSTHOOK: query: create external table vector_decimal64_mul_decimal64column(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,1), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vector_decimal64_mul_decimal64column +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_decimal64column +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@vector_decimal64_mul_decimal64column +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_decimal64column +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@vector_decimal64_mul_decimal64column +PREHOOK: query: create table vector_decimal64_mul_decimal64column_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,1), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) stored as ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vector_decimal64_mul_decimal64column_tmp +POSTHOOK: query: create table vector_decimal64_mul_decimal64column_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,1), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) stored as ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vector_decimal64_mul_decimal64column_tmp +PREHOOK: query: insert into table vector_decimal64_mul_decimal64column_tmp select * from vector_decimal64_mul_decimal64column +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64column +PREHOOK: Output: default@vector_decimal64_mul_decimal64column_tmp +POSTHOOK: query: insert into table vector_decimal64_mul_decimal64column_tmp select * from vector_decimal64_mul_decimal64column +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64column +POSTHOOK: Output: default@vector_decimal64_mul_decimal64column_tmp +POSTHOOK: Lineage: vector_decimal64_mul_decimal64column_tmp.ss_ext_discount_amt SIMPLE [(vector_decimal64_mul_decimal64column)vector_decimal64_mul_decimal64column.FieldSchema(name:ss_ext_discount_amt, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64column_tmp.ss_ext_list_price SIMPLE [(vector_decimal64_mul_decimal64column)vector_decimal64_mul_decimal64column.FieldSchema(name:ss_ext_list_price, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64column_tmp.ss_ext_sales_price SIMPLE [(vector_decimal64_mul_decimal64column)vector_decimal64_mul_decimal64column.FieldSchema(name:ss_ext_sales_price, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64column_tmp.ss_ext_wholesale_cost SIMPLE [(vector_decimal64_mul_decimal64column)vector_decimal64_mul_decimal64column.FieldSchema(name:ss_ext_wholesale_cost, type:decimal(7,1), comment:null), ] +PREHOOK: query: explain vectorization detail select sum(ss_ext_list_price+ss_ext_wholesale_cost) from vector_decimal64_mul_decimal64column_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64column_tmp +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail select sum(ss_ext_list_price+ss_ext_wholesale_cost) from vector_decimal64_mul_decimal64column_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64column_tmp +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: vector_decimal64_mul_decimal64column_tmp + Statistics: Num rows: 1000 Data size: 224000 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:ss_ext_list_price:decimal(7,2)/DECIMAL_64, 1:ss_ext_wholesale_cost:decimal(7,1)/DECIMAL_64, 2:ss_ext_discount_amt:decimal(7,2)/DECIMAL_64, 3:ss_ext_sales_price:decimal(7,2)/DECIMAL_64, 4:ROW__ID:struct] + Select Operator + expressions: (ss_ext_list_price + ss_ext_wholesale_cost) (type: decimal(9,2)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [6] + selectExpressions: Decimal64ColAddDecimal64Column(col 0:decimal(7,2)/DECIMAL_64, col 5:decimal(8,2)/DECIMAL_64)(children: Decimal64ColScaleUp(col 1:decimal(7,1)/DECIMAL_64, decimal64Val 10, decimalVal 10) -> 5:decimal(8,2)/DECIMAL_64) -> 6:decimal(9,2)/DECIMAL_64 + Statistics: Num rows: 1000 Data size: 224000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal64ToDecimal(col 6:decimal(9,2)/DECIMAL_64) -> decimal(19,2) + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 0:decimal(19,2) + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(19,2)) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 4 + includeColumns: [0, 1] + dataColumns: ss_ext_list_price:decimal(7,2)/DECIMAL_64, ss_ext_wholesale_cost:decimal(7,1)/DECIMAL_64, ss_ext_discount_amt:decimal(7,2)/DECIMAL_64, ss_ext_sales_price:decimal(7,2)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [decimal(8,2)/DECIMAL_64, decimal(9,2)/DECIMAL_64] + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: + reduceColumnSortOrder: + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + dataColumns: VALUE._col0:decimal(19,2) + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal(col 0:decimal(19,2)) -> decimal(19,2) + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumnNums: [0] + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select sum(ss_ext_list_price+ss_ext_wholesale_cost) from vector_decimal64_mul_decimal64column_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64column_tmp +#### A masked pattern was here #### +POSTHOOK: query: select sum(ss_ext_list_price+ss_ext_wholesale_cost) from vector_decimal64_mul_decimal64column_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64column_tmp +#### A masked pattern was here #### +2001000.00 diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal64_mul_intcolumn.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal64_mul_intcolumn.q.out new file mode 100644 index 0000000000..612feccaeb --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/vector_decimal64_mul_intcolumn.q.out @@ -0,0 +1,306 @@ +PREHOOK: query: create external table vector_decimal64_mul_intcolumn(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(19,1), ss_ext_discount_amt int, ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vector_decimal64_mul_intcolumn +POSTHOOK: query: create external table vector_decimal64_mul_intcolumn(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(19,1), ss_ext_discount_amt int, ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vector_decimal64_mul_intcolumn +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_intcolumn +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@vector_decimal64_mul_intcolumn +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_intcolumn +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@vector_decimal64_mul_intcolumn +PREHOOK: query: create table vector_decimal64_mul_intcolumn_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(19,1), ss_ext_discount_amt int, ss_ext_sales_price decimal(7,2)) stored as ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vector_decimal64_mul_intcolumn_tmp +POSTHOOK: query: create table vector_decimal64_mul_intcolumn_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(19,1), ss_ext_discount_amt int, ss_ext_sales_price decimal(7,2)) stored as ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vector_decimal64_mul_intcolumn_tmp +PREHOOK: query: insert into table vector_decimal64_mul_intcolumn_tmp select * from vector_decimal64_mul_intcolumn +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_intcolumn +PREHOOK: Output: default@vector_decimal64_mul_intcolumn_tmp +POSTHOOK: query: insert into table vector_decimal64_mul_intcolumn_tmp select * from vector_decimal64_mul_intcolumn +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_intcolumn +POSTHOOK: Output: default@vector_decimal64_mul_intcolumn_tmp +POSTHOOK: Lineage: vector_decimal64_mul_intcolumn_tmp.ss_ext_discount_amt SIMPLE [(vector_decimal64_mul_intcolumn)vector_decimal64_mul_intcolumn.FieldSchema(name:ss_ext_discount_amt, type:int, comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_intcolumn_tmp.ss_ext_list_price SIMPLE [(vector_decimal64_mul_intcolumn)vector_decimal64_mul_intcolumn.FieldSchema(name:ss_ext_list_price, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_intcolumn_tmp.ss_ext_sales_price SIMPLE [(vector_decimal64_mul_intcolumn)vector_decimal64_mul_intcolumn.FieldSchema(name:ss_ext_sales_price, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_intcolumn_tmp.ss_ext_wholesale_cost SIMPLE [(vector_decimal64_mul_intcolumn)vector_decimal64_mul_intcolumn.FieldSchema(name:ss_ext_wholesale_cost, type:decimal(19,1), comment:null), ] +PREHOOK: query: explain vectorization detail select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_intcolumn_tmp +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_intcolumn_tmp +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: vector_decimal64_mul_intcolumn_tmp + Statistics: Num rows: 1000 Data size: 116000 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:ss_ext_list_price:decimal(7,2)/DECIMAL_64, 1:ss_ext_wholesale_cost:decimal(19,1), 2:ss_ext_discount_amt:int, 3:ss_ext_sales_price:decimal(7,2)/DECIMAL_64, 4:ROW__ID:struct] + Select Operator + expressions: (ss_ext_list_price * CAST( ss_ext_discount_amt AS decimal(10,0))) (type: decimal(18,2)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [6] + selectExpressions: Decimal64ColMultiplyDecimal64Column(col 0:decimal(7,2)/DECIMAL_64, col 5:decimal(10,0)/DECIMAL_64)(children: CastLongToDecimal64(col 2:int) -> 5:decimal(10,0)/DECIMAL_64) -> 6:decimal(18,2)/DECIMAL_64 + Statistics: Num rows: 1000 Data size: 116000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal64ToDecimal(col 6:decimal(18,2)/DECIMAL_64) -> decimal(28,2) + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 0:decimal(28,2) + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(28,2)) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 4 + includeColumns: [0, 2] + dataColumns: ss_ext_list_price:decimal(7,2)/DECIMAL_64, ss_ext_wholesale_cost:decimal(19,1), ss_ext_discount_amt:int, ss_ext_sales_price:decimal(7,2)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [decimal(10,0)/DECIMAL_64, decimal(18,2)/DECIMAL_64] + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: + reduceColumnSortOrder: + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + dataColumns: VALUE._col0:decimal(28,2) + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal(col 0:decimal(28,2)) -> decimal(28,2) + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumnNums: [0] + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_intcolumn_tmp +#### A masked pattern was here #### +POSTHOOK: query: select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_intcolumn_tmp +#### A masked pattern was here #### +984383500.00 +PREHOOK: query: explain vectorization detail select sum(ss_ext_wholesale_cost*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_intcolumn_tmp +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail select sum(ss_ext_wholesale_cost*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_intcolumn_tmp +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: vector_decimal64_mul_intcolumn_tmp + Statistics: Num rows: 1000 Data size: 116000 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:ss_ext_list_price:decimal(7,2)/DECIMAL_64, 1:ss_ext_wholesale_cost:decimal(19,1), 2:ss_ext_discount_amt:int, 3:ss_ext_sales_price:decimal(7,2)/DECIMAL_64, 4:ROW__ID:struct] + Select Operator + expressions: (ss_ext_wholesale_cost * CAST( ss_ext_discount_amt AS decimal(10,0))) (type: decimal(30,1)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [6] + selectExpressions: DecimalColMultiplyDecimalColumn(col 1:decimal(19,1), col 5:decimal(10,0))(children: CastLongToDecimal(col 2:int) -> 5:decimal(10,0)) -> 6:decimal(30,1) + Statistics: Num rows: 1000 Data size: 116000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal(col 6:decimal(30,1)) -> decimal(38,1) + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 0:decimal(38,1) + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(38,1)) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 4 + includeColumns: [1, 2] + dataColumns: ss_ext_list_price:decimal(7,2)/DECIMAL_64, ss_ext_wholesale_cost:decimal(19,1), ss_ext_discount_amt:int, ss_ext_sales_price:decimal(7,2)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [decimal(10,0), decimal(30,1)] + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: + reduceColumnSortOrder: + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + dataColumns: VALUE._col0:decimal(38,1) + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal(col 0:decimal(38,1)) -> decimal(38,1) + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumnNums: [0] + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select sum(ss_ext_wholesale_cost*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_intcolumn_tmp +#### A masked pattern was here #### +POSTHOOK: query: select sum(ss_ext_wholesale_cost*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_intcolumn_tmp +#### A masked pattern was here #### +383883500.0 diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out index 8500562893..32859ddb0a 100644 --- a/ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out @@ -258,8 +258,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [7, 10, 13, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36] - selectExpressions: DecimalColAddDecimalColumn(col 5:decimal(10,3), col 6:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 5:decimal(10,3), ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 6:decimal(7,2)) -> 7:decimal(11,3), DecimalColSubtractDecimalColumn(col 8:decimal(10,3), col 37:decimal(9,2))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 8:decimal(10,3), ConvertDecimal64ToDecimal(col 9:decimal(9,2)/DECIMAL_64)(children: Decimal64ScalarMultiplyDecimal64ColumnUnscaled(decimal64Val 2, decimalVal 2, col 2:decimal(7,2)/DECIMAL_64) -> 9:decimal(9,2)/DECIMAL_64) -> 37:decimal(9,2)) -> 10:decimal(11,3), DecimalColDivideDecimalColumn(col 38:decimal(11,3), col 12:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 11:decimal(11,3)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,3)/DECIMAL_64, decimal64Val 2340, decimalVal 2.34) -> 11:decimal(11,3)/DECIMAL_64) -> 38:decimal(11,3), ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 12:decimal(7,2)) -> 13:decimal(21,11), DecimalColMultiplyDecimalColumn(col 14:decimal(10,3), col 39:decimal(12,6))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 14:decimal(10,3), ConvertDecimal64ToDecimal(col 15:decimal(12,6)/DECIMAL_64)(children: Decimal64ColDivideDecimal64Scalar(col 2:decimal(7,2)/DECIMAL_64, decimal64Val 340, decimalVal 3.4) -> 15:decimal(12,6)/DECIMAL_64) -> 39:decimal(12,6)) -> 16:decimal(23,9), DecimalColModuloDecimalScalar(col 17:decimal(10,3), val 10)(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 17:decimal(10,3)) -> 18:decimal(5,3), CastDecimalToLong(col 19:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 19:decimal(10,3)) -> 20:int, CastDecimalToLong(col 21:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 21:decimal(7,2)) -> 22:smallint, CastDecimalToLong(col 23:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 23:decimal(7,2)) -> 24:tinyint, CastDecimalToLong(col 25:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 25:decimal(10,3)) -> 26:bigint, CastDecimalToBoolean(col 27:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 27:decimal(10,3)) -> 28:boolean, CastDecimalToDouble(col 29:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 29:decimal(7,2)) -> 30:double, CastDecimalToFloat(col 31:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 31:decimal(10,3)) -> 32:float, CastDecimalToString(col 33:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 33:decimal(7,2)) -> 34:string, CastDecimalToTimestamp(col 35:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 35:decimal(10,3)) -> 36:timestamp + projectedOutputColumnNums: [6, 9, 12, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35] + selectExpressions: Decimal64ColAddDecimal64Column(col 1:decimal(10,3)/DECIMAL_64, col 5:decimal(8,3)/DECIMAL_64)(children: Decimal64ColScaleUp(col 2:decimal(7,2)/DECIMAL_64, decimal64Val 10, decimalVal 10) -> 5:decimal(8,3)/DECIMAL_64) -> 6:decimal(11,3)/DECIMAL_64, Decimal64ColSubtractDecimal64Column(col 1:decimal(10,3)/DECIMAL_64, col 8:decimal(10,3)/DECIMAL_64)(children: Decimal64ColScaleUp(col 7:decimal(9,2)/DECIMAL_64, decimal64Val 10, decimalVal 10)(children: Decimal64ScalarMultiplyDecimal64ColumnUnscaled(decimal64Val 2, decimalVal 2, col 2:decimal(7,2)/DECIMAL_64) -> 7:decimal(9,2)/DECIMAL_64) -> 8:decimal(10,3)/DECIMAL_64) -> 9:decimal(11,3)/DECIMAL_64, DecimalColDivideDecimalColumn(col 36:decimal(11,3), col 11:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 10:decimal(11,3)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,3)/DECIMAL_64, decimal64Val 2340, decimalVal 2.34) -> 10:decimal(11,3)/DECIMAL_64) -> 36:decimal(11,3), ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 11:decimal(7,2)) -> 12:decimal(21,11), DecimalColMultiplyDecimalColumn(col 13:decimal(10,3), col 37:decimal(12,6))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 13:decimal(10,3), ConvertDecimal64ToDecimal(col 14:decimal(12,6)/DECIMAL_64)(children: Decimal64ColDivideDecimal64Scalar(col 2:decimal(7,2)/DECIMAL_64, decimal64Val 340, decimalVal 3.4) -> 14:decimal(12,6)/DECIMAL_64) -> 37:decimal(12,6)) -> 15:decimal(23,9), DecimalColModuloDecimalScalar(col 16:decimal(10,3), val 10)(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 16:decimal(10,3)) -> 17:decimal(5,3), CastDecimalToLong(col 18:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 18:decimal(10,3)) -> 19:int, CastDecimalToLong(col 20:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 20:decimal(7,2)) -> 21:smallint, CastDecimalToLong(col 22:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 22:decimal(7,2)) -> 23:tinyint, CastDecimalToLong(col 24:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 24:decimal(10,3)) -> 25:bigint, CastDecimalToBoolean(col 26:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 26:decimal(10,3)) -> 27:boolean, CastDecimalToDouble(col 28:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 28:decimal(7,2)) -> 29:double, CastDecimalToFloat(col 30:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 30:decimal(10,3)) -> 31:float, CastDecimalToString(col 32:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 32:decimal(7,2)) -> 33:string, CastDecimalToTimestamp(col 34:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 34:decimal(10,3)) -> 35:timestamp Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(11,3)), _col1 (type: decimal(11,3)), _col2 (type: decimal(21,11)), _col3 (type: decimal(23,9)), _col4 (type: decimal(5,3)), _col5 (type: int), _col6 (type: smallint), _col7 (type: tinyint), _col8 (type: bigint), _col9 (type: boolean), _col10 (type: double), _col11 (type: float), _col12 (type: string), _col13 (type: timestamp) @@ -267,7 +267,7 @@ STAGE PLANS: sort order: ++++++++++++++ Reduce Sink Vectorization: className: VectorReduceSinkObjectHashOperator - keyColumns: 7:decimal(11,3), 10:decimal(11,3), 13:decimal(21,11), 16:decimal(23,9), 18:decimal(5,3), 20:int, 22:smallint, 24:tinyint, 26:bigint, 28:boolean, 30:double, 32:float, 34:string, 36:timestamp + keyColumns: 6:decimal(11,3), 9:decimal(11,3), 12:decimal(21,11), 15:decimal(23,9), 17:decimal(5,3), 19:int, 21:smallint, 23:tinyint, 25:bigint, 27:boolean, 29:double, 31:float, 33:string, 35:timestamp native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE @@ -288,7 +288,7 @@ STAGE PLANS: includeColumns: [0, 1, 2] dataColumns: cdouble:double, cdecimal1:decimal(10,3)/DECIMAL_64, cdecimal2:decimal(7,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(10,3), decimal(10,3), decimal(7,2), decimal(11,3), decimal(10,3), decimal(9,2)/DECIMAL_64, decimal(11,3), decimal(11,3)/DECIMAL_64, decimal(7,2), decimal(21,11), decimal(10,3), decimal(12,6)/DECIMAL_64, decimal(23,9), decimal(10,3), decimal(5,3), decimal(10,3), bigint, decimal(7,2), bigint, decimal(7,2), bigint, decimal(10,3), bigint, decimal(10,3), bigint, decimal(7,2), double, decimal(10,3), double, decimal(7,2), string, decimal(10,3), timestamp, decimal(9,2), decimal(11,3), decimal(12,6)] + scratchColumnTypeNames: [decimal(10,3), decimal(8,3)/DECIMAL_64, decimal(11,3)/DECIMAL_64, decimal(9,2)/DECIMAL_64, decimal(10,3)/DECIMAL_64, decimal(11,3)/DECIMAL_64, decimal(11,3)/DECIMAL_64, decimal(7,2), decimal(21,11), decimal(10,3), decimal(12,6)/DECIMAL_64, decimal(23,9), decimal(10,3), decimal(5,3), decimal(10,3), bigint, decimal(7,2), bigint, decimal(7,2), bigint, decimal(10,3), bigint, decimal(10,3), bigint, decimal(7,2), double, decimal(10,3), double, decimal(7,2), string, decimal(10,3), timestamp, decimal(11,3), decimal(12,6)] Reducer 2 Execution mode: vectorized, llap Reduce Vectorization: diff --git a/ql/src/test/results/clientpositive/vector_decimal64_add_decimal64column_diffscale.q.out b/ql/src/test/results/clientpositive/vector_decimal64_add_decimal64column_diffscale.q.out new file mode 100644 index 0000000000..3847195c4f --- /dev/null +++ b/ql/src/test/results/clientpositive/vector_decimal64_add_decimal64column_diffscale.q.out @@ -0,0 +1,143 @@ +PREHOOK: query: create external table vector_decimal64_mul_decimal64column(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,1), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vector_decimal64_mul_decimal64column +POSTHOOK: query: create external table vector_decimal64_mul_decimal64column(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,1), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vector_decimal64_mul_decimal64column +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_decimal64column +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@vector_decimal64_mul_decimal64column +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_decimal64column +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@vector_decimal64_mul_decimal64column +PREHOOK: query: create table vector_decimal64_mul_decimal64column_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,1), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) stored as ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vector_decimal64_mul_decimal64column_tmp +POSTHOOK: query: create table vector_decimal64_mul_decimal64column_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,1), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) stored as ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vector_decimal64_mul_decimal64column_tmp +PREHOOK: query: insert into table vector_decimal64_mul_decimal64column_tmp select * from vector_decimal64_mul_decimal64column +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64column +PREHOOK: Output: default@vector_decimal64_mul_decimal64column_tmp +POSTHOOK: query: insert into table vector_decimal64_mul_decimal64column_tmp select * from vector_decimal64_mul_decimal64column +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64column +POSTHOOK: Output: default@vector_decimal64_mul_decimal64column_tmp +POSTHOOK: Lineage: vector_decimal64_mul_decimal64column_tmp.ss_ext_discount_amt SIMPLE [(vector_decimal64_mul_decimal64column)vector_decimal64_mul_decimal64column.FieldSchema(name:ss_ext_discount_amt, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64column_tmp.ss_ext_list_price SIMPLE [(vector_decimal64_mul_decimal64column)vector_decimal64_mul_decimal64column.FieldSchema(name:ss_ext_list_price, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64column_tmp.ss_ext_sales_price SIMPLE [(vector_decimal64_mul_decimal64column)vector_decimal64_mul_decimal64column.FieldSchema(name:ss_ext_sales_price, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64column_tmp.ss_ext_wholesale_cost SIMPLE [(vector_decimal64_mul_decimal64column)vector_decimal64_mul_decimal64column.FieldSchema(name:ss_ext_wholesale_cost, type:decimal(7,1), comment:null), ] +PREHOOK: query: explain vectorization detail select sum(ss_ext_list_price+ss_ext_wholesale_cost) from vector_decimal64_mul_decimal64column_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64column_tmp +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail select sum(ss_ext_list_price+ss_ext_wholesale_cost) from vector_decimal64_mul_decimal64column_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64column_tmp +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: vector_decimal64_mul_decimal64column_tmp + Statistics: Num rows: 1000 Data size: 224000 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:ss_ext_list_price:decimal(7,2)/DECIMAL_64, 1:ss_ext_wholesale_cost:decimal(7,1)/DECIMAL_64, 2:ss_ext_discount_amt:decimal(7,2)/DECIMAL_64, 3:ss_ext_sales_price:decimal(7,2)/DECIMAL_64, 4:ROW__ID:struct] + Select Operator + expressions: (ss_ext_list_price + ss_ext_wholesale_cost) (type: decimal(9,2)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [6] + selectExpressions: Decimal64ColAddDecimal64Column(col 0:decimal(7,2)/DECIMAL_64, col 5:decimal(8,2)/DECIMAL_64)(children: Decimal64ColScaleUp(col 1:decimal(7,1)/DECIMAL_64, decimal64Val 10, decimalVal 10) -> 5:decimal(8,2)/DECIMAL_64) -> 6:decimal(9,2)/DECIMAL_64 + Statistics: Num rows: 1000 Data size: 224000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal64ToDecimal(col 6:decimal(9,2)/DECIMAL_64) -> decimal(19,2) + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkOperator + native: false + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(19,2)) + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 4 + includeColumns: [0, 1] + dataColumns: ss_ext_list_price:decimal(7,2)/DECIMAL_64, ss_ext_wholesale_cost:decimal(7,1)/DECIMAL_64, ss_ext_discount_amt:decimal(7,2)/DECIMAL_64, ss_ext_sales_price:decimal(7,2)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [decimal(8,2)/DECIMAL_64, decimal(9,2)/DECIMAL_64] + Reduce Vectorization: + enabled: false + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true + enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select sum(ss_ext_list_price+ss_ext_wholesale_cost) from vector_decimal64_mul_decimal64column_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64column_tmp +#### A masked pattern was here #### +POSTHOOK: query: select sum(ss_ext_list_price+ss_ext_wholesale_cost) from vector_decimal64_mul_decimal64column_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64column_tmp +#### A masked pattern was here #### +2001000.00 diff --git a/ql/src/test/results/clientpositive/vector_decimal64_mul_intcolumn.q.out b/ql/src/test/results/clientpositive/vector_decimal64_mul_intcolumn.q.out new file mode 100644 index 0000000000..e4fd531d3b --- /dev/null +++ b/ql/src/test/results/clientpositive/vector_decimal64_mul_intcolumn.q.out @@ -0,0 +1,250 @@ +PREHOOK: query: create external table vector_decimal64_mul_intcolumn(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(19,1), ss_ext_discount_amt int, ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vector_decimal64_mul_intcolumn +POSTHOOK: query: create external table vector_decimal64_mul_intcolumn(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(19,1), ss_ext_discount_amt int, ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vector_decimal64_mul_intcolumn +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_intcolumn +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@vector_decimal64_mul_intcolumn +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_intcolumn +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@vector_decimal64_mul_intcolumn +PREHOOK: query: create table vector_decimal64_mul_intcolumn_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(19,1), ss_ext_discount_amt int, ss_ext_sales_price decimal(7,2)) stored as ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vector_decimal64_mul_intcolumn_tmp +POSTHOOK: query: create table vector_decimal64_mul_intcolumn_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(19,1), ss_ext_discount_amt int, ss_ext_sales_price decimal(7,2)) stored as ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vector_decimal64_mul_intcolumn_tmp +PREHOOK: query: insert into table vector_decimal64_mul_intcolumn_tmp select * from vector_decimal64_mul_intcolumn +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_intcolumn +PREHOOK: Output: default@vector_decimal64_mul_intcolumn_tmp +POSTHOOK: query: insert into table vector_decimal64_mul_intcolumn_tmp select * from vector_decimal64_mul_intcolumn +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_intcolumn +POSTHOOK: Output: default@vector_decimal64_mul_intcolumn_tmp +POSTHOOK: Lineage: vector_decimal64_mul_intcolumn_tmp.ss_ext_discount_amt SIMPLE [(vector_decimal64_mul_intcolumn)vector_decimal64_mul_intcolumn.FieldSchema(name:ss_ext_discount_amt, type:int, comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_intcolumn_tmp.ss_ext_list_price SIMPLE [(vector_decimal64_mul_intcolumn)vector_decimal64_mul_intcolumn.FieldSchema(name:ss_ext_list_price, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_intcolumn_tmp.ss_ext_sales_price SIMPLE [(vector_decimal64_mul_intcolumn)vector_decimal64_mul_intcolumn.FieldSchema(name:ss_ext_sales_price, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_intcolumn_tmp.ss_ext_wholesale_cost SIMPLE [(vector_decimal64_mul_intcolumn)vector_decimal64_mul_intcolumn.FieldSchema(name:ss_ext_wholesale_cost, type:decimal(19,1), comment:null), ] +PREHOOK: query: explain vectorization detail select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_intcolumn_tmp +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_intcolumn_tmp +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: vector_decimal64_mul_intcolumn_tmp + Statistics: Num rows: 1000 Data size: 116000 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:ss_ext_list_price:decimal(7,2)/DECIMAL_64, 1:ss_ext_wholesale_cost:decimal(19,1), 2:ss_ext_discount_amt:int, 3:ss_ext_sales_price:decimal(7,2)/DECIMAL_64, 4:ROW__ID:struct] + Select Operator + expressions: (ss_ext_list_price * CAST( ss_ext_discount_amt AS decimal(10,0))) (type: decimal(18,2)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [6] + selectExpressions: Decimal64ColMultiplyDecimal64Column(col 0:decimal(7,2)/DECIMAL_64, col 5:decimal(10,0)/DECIMAL_64)(children: CastLongToDecimal64(col 2:int) -> 5:decimal(10,0)/DECIMAL_64) -> 6:decimal(18,2)/DECIMAL_64 + Statistics: Num rows: 1000 Data size: 116000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal64ToDecimal(col 6:decimal(18,2)/DECIMAL_64) -> decimal(28,2) + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkOperator + native: false + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(28,2)) + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 4 + includeColumns: [0, 2] + dataColumns: ss_ext_list_price:decimal(7,2)/DECIMAL_64, ss_ext_wholesale_cost:decimal(19,1), ss_ext_discount_amt:int, ss_ext_sales_price:decimal(7,2)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [decimal(10,0)/DECIMAL_64, decimal(18,2)/DECIMAL_64] + Reduce Vectorization: + enabled: false + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true + enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_intcolumn_tmp +#### A masked pattern was here #### +POSTHOOK: query: select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_intcolumn_tmp +#### A masked pattern was here #### +984383500.00 +PREHOOK: query: explain vectorization detail select sum(ss_ext_wholesale_cost*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_intcolumn_tmp +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail select sum(ss_ext_wholesale_cost*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_intcolumn_tmp +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: vector_decimal64_mul_intcolumn_tmp + Statistics: Num rows: 1000 Data size: 116000 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:ss_ext_list_price:decimal(7,2)/DECIMAL_64, 1:ss_ext_wholesale_cost:decimal(19,1), 2:ss_ext_discount_amt:int, 3:ss_ext_sales_price:decimal(7,2)/DECIMAL_64, 4:ROW__ID:struct] + Select Operator + expressions: (ss_ext_wholesale_cost * CAST( ss_ext_discount_amt AS decimal(10,0))) (type: decimal(30,1)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [6] + selectExpressions: DecimalColMultiplyDecimalColumn(col 1:decimal(19,1), col 5:decimal(10,0))(children: CastLongToDecimal(col 2:int) -> 5:decimal(10,0)) -> 6:decimal(30,1) + Statistics: Num rows: 1000 Data size: 116000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal(col 6:decimal(30,1)) -> decimal(38,1) + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkOperator + native: false + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(38,1)) + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 4 + includeColumns: [1, 2] + dataColumns: ss_ext_list_price:decimal(7,2)/DECIMAL_64, ss_ext_wholesale_cost:decimal(19,1), ss_ext_discount_amt:int, ss_ext_sales_price:decimal(7,2)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [decimal(10,0), decimal(30,1)] + Reduce Vectorization: + enabled: false + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true + enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select sum(ss_ext_wholesale_cost*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_intcolumn_tmp +#### A masked pattern was here #### +POSTHOOK: query: select sum(ss_ext_wholesale_cost*ss_ext_discount_amt) from vector_decimal64_mul_intcolumn_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_intcolumn_tmp +#### A masked pattern was here #### +383883500.0 diff --git a/ql/src/test/results/clientpositive/vector_decimal_expressions.q.out b/ql/src/test/results/clientpositive/vector_decimal_expressions.q.out index 0cfbb6f68e..e0f02a783a 100644 --- a/ql/src/test/results/clientpositive/vector_decimal_expressions.q.out +++ b/ql/src/test/results/clientpositive/vector_decimal_expressions.q.out @@ -224,8 +224,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [7, 10, 13, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36] - selectExpressions: DecimalColAddDecimalColumn(col 5:decimal(10,3), col 6:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 5:decimal(10,3), ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 6:decimal(7,2)) -> 7:decimal(11,3), DecimalColSubtractDecimalColumn(col 8:decimal(10,3), col 37:decimal(9,2))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 8:decimal(10,3), ConvertDecimal64ToDecimal(col 9:decimal(9,2)/DECIMAL_64)(children: Decimal64ScalarMultiplyDecimal64ColumnUnscaled(decimal64Val 2, decimalVal 2, col 2:decimal(7,2)/DECIMAL_64) -> 9:decimal(9,2)/DECIMAL_64) -> 37:decimal(9,2)) -> 10:decimal(11,3), DecimalColDivideDecimalColumn(col 38:decimal(11,3), col 12:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 11:decimal(11,3)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,3)/DECIMAL_64, decimal64Val 2340, decimalVal 2.34) -> 11:decimal(11,3)/DECIMAL_64) -> 38:decimal(11,3), ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 12:decimal(7,2)) -> 13:decimal(21,11), DecimalColMultiplyDecimalColumn(col 14:decimal(10,3), col 39:decimal(12,6))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 14:decimal(10,3), ConvertDecimal64ToDecimal(col 15:decimal(12,6)/DECIMAL_64)(children: Decimal64ColDivideDecimal64Scalar(col 2:decimal(7,2)/DECIMAL_64, decimal64Val 340, decimalVal 3.4) -> 15:decimal(12,6)/DECIMAL_64) -> 39:decimal(12,6)) -> 16:decimal(23,9), DecimalColModuloDecimalScalar(col 17:decimal(10,3), val 10)(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 17:decimal(10,3)) -> 18:decimal(5,3), CastDecimalToLong(col 19:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 19:decimal(10,3)) -> 20:int, CastDecimalToLong(col 21:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 21:decimal(7,2)) -> 22:smallint, CastDecimalToLong(col 23:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 23:decimal(7,2)) -> 24:tinyint, CastDecimalToLong(col 25:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 25:decimal(10,3)) -> 26:bigint, CastDecimalToBoolean(col 27:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 27:decimal(10,3)) -> 28:boolean, CastDecimalToDouble(col 29:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 29:decimal(7,2)) -> 30:double, CastDecimalToFloat(col 31:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 31:decimal(10,3)) -> 32:float, CastDecimalToString(col 33:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 33:decimal(7,2)) -> 34:string, CastDecimalToTimestamp(col 35:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 35:decimal(10,3)) -> 36:timestamp + projectedOutputColumnNums: [6, 9, 12, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35] + selectExpressions: Decimal64ColAddDecimal64Column(col 1:decimal(10,3)/DECIMAL_64, col 5:decimal(8,3)/DECIMAL_64)(children: Decimal64ColScaleUp(col 2:decimal(7,2)/DECIMAL_64, decimal64Val 10, decimalVal 10) -> 5:decimal(8,3)/DECIMAL_64) -> 6:decimal(11,3)/DECIMAL_64, Decimal64ColSubtractDecimal64Column(col 1:decimal(10,3)/DECIMAL_64, col 8:decimal(10,3)/DECIMAL_64)(children: Decimal64ColScaleUp(col 7:decimal(9,2)/DECIMAL_64, decimal64Val 10, decimalVal 10)(children: Decimal64ScalarMultiplyDecimal64ColumnUnscaled(decimal64Val 2, decimalVal 2, col 2:decimal(7,2)/DECIMAL_64) -> 7:decimal(9,2)/DECIMAL_64) -> 8:decimal(10,3)/DECIMAL_64) -> 9:decimal(11,3)/DECIMAL_64, DecimalColDivideDecimalColumn(col 36:decimal(11,3), col 11:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 10:decimal(11,3)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,3)/DECIMAL_64, decimal64Val 2340, decimalVal 2.34) -> 10:decimal(11,3)/DECIMAL_64) -> 36:decimal(11,3), ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 11:decimal(7,2)) -> 12:decimal(21,11), DecimalColMultiplyDecimalColumn(col 13:decimal(10,3), col 37:decimal(12,6))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 13:decimal(10,3), ConvertDecimal64ToDecimal(col 14:decimal(12,6)/DECIMAL_64)(children: Decimal64ColDivideDecimal64Scalar(col 2:decimal(7,2)/DECIMAL_64, decimal64Val 340, decimalVal 3.4) -> 14:decimal(12,6)/DECIMAL_64) -> 37:decimal(12,6)) -> 15:decimal(23,9), DecimalColModuloDecimalScalar(col 16:decimal(10,3), val 10)(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 16:decimal(10,3)) -> 17:decimal(5,3), CastDecimalToLong(col 18:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 18:decimal(10,3)) -> 19:int, CastDecimalToLong(col 20:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 20:decimal(7,2)) -> 21:smallint, CastDecimalToLong(col 22:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 22:decimal(7,2)) -> 23:tinyint, CastDecimalToLong(col 24:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 24:decimal(10,3)) -> 25:bigint, CastDecimalToBoolean(col 26:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 26:decimal(10,3)) -> 27:boolean, CastDecimalToDouble(col 28:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 28:decimal(7,2)) -> 29:double, CastDecimalToFloat(col 30:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 30:decimal(10,3)) -> 31:float, CastDecimalToString(col 32:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 32:decimal(7,2)) -> 33:string, CastDecimalToTimestamp(col 34:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 34:decimal(10,3)) -> 35:timestamp Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(11,3)), _col1 (type: decimal(11,3)), _col2 (type: decimal(21,11)), _col3 (type: decimal(23,9)), _col4 (type: decimal(5,3)), _col5 (type: int), _col6 (type: smallint), _col7 (type: tinyint), _col8 (type: bigint), _col9 (type: boolean), _col10 (type: double), _col11 (type: float), _col12 (type: string), _col13 (type: timestamp) @@ -253,7 +253,7 @@ STAGE PLANS: includeColumns: [0, 1, 2] dataColumns: cdouble:double, cdecimal1:decimal(10,3)/DECIMAL_64, cdecimal2:decimal(7,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(10,3), decimal(10,3), decimal(7,2), decimal(11,3), decimal(10,3), decimal(9,2)/DECIMAL_64, decimal(11,3), decimal(11,3)/DECIMAL_64, decimal(7,2), decimal(21,11), decimal(10,3), decimal(12,6)/DECIMAL_64, decimal(23,9), decimal(10,3), decimal(5,3), decimal(10,3), bigint, decimal(7,2), bigint, decimal(7,2), bigint, decimal(10,3), bigint, decimal(10,3), bigint, decimal(7,2), double, decimal(10,3), double, decimal(7,2), string, decimal(10,3), timestamp, decimal(9,2), decimal(11,3), decimal(12,6)] + scratchColumnTypeNames: [decimal(10,3), decimal(8,3)/DECIMAL_64, decimal(11,3)/DECIMAL_64, decimal(9,2)/DECIMAL_64, decimal(10,3)/DECIMAL_64, decimal(11,3)/DECIMAL_64, decimal(11,3)/DECIMAL_64, decimal(7,2), decimal(21,11), decimal(10,3), decimal(12,6)/DECIMAL_64, decimal(23,9), decimal(10,3), decimal(5,3), decimal(10,3), bigint, decimal(7,2), bigint, decimal(7,2), bigint, decimal(10,3), bigint, decimal(10,3), bigint, decimal(7,2), double, decimal(10,3), double, decimal(7,2), string, decimal(10,3), timestamp, decimal(11,3), decimal(12,6)] Reduce Vectorization: enabled: false enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true diff --git a/vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java b/vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java index 7b6b6eeb75..cc413c828f 100644 --- a/vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java +++ b/vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java @@ -322,6 +322,8 @@ {"Decimal64ColumnDivideDecimal64Scalar", "Divide", "/"}, {"Decimal64ColumnDivideDecimal64Column", "Divide", "/"}, + {"Decimal64ColumnScaleUp", "ScaleUp", "*"}, + {"ColumnCompareScalar", "Equal", "long", "long", "=="}, {"ColumnCompareScalar", "Equal", "long", "double", "=="}, {"ColumnCompareScalar", "Equal", "double", "double", "=="}, @@ -1424,6 +1426,8 @@ private void generate() throws Exception { generateDecimal64ColumnArithmeticDecimal64Scalar(tdesc); } else if (tdesc[0].equals("Decimal64ColumnDivideDecimal64Column")) { generateDecimal64ColumnArithmeticDecimal64Column(tdesc); + } else if(tdesc[0].equals("Decimal64ColumnScaleUp")) { + generateDecimal64ColumnScaleUp(tdesc); } else if (tdesc[0].equals("ColumnUnaryMinus")) { generateColumnUnaryMinus(tdesc); } else if (tdesc[0].equals("ColumnUnaryFunc")) { @@ -2563,6 +2567,13 @@ private void generateDecimal64ColumnArithmeticDecimal64Column(String[] tdesc) th generateDecimal64ColumnArithmetic(tdesc, className, /* parentClassName */ null); } + private void generateDecimal64ColumnScaleUp(String[] tdesc) throws IOException { + String operatorName = tdesc[1]; + String className = "Decimal64Col" + operatorName; + String parentClassName = "Decimal64ColMultiplyDecimal64Scalar"; + generateDecimal64ColumnArithmetic(tdesc, className, parentClassName); + } + private void generateDecimal64ColumnArithmetic(String[] tdesc, String className, String parentClassName) throws IOException {