diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index cea0c8d306..023dbb9b1d 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -262,6 +262,8 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\ vector_date_1.q,\ vector_decimal64_div_decimal64scalar.q,\ vector_decimal64_div_decimal64column.q,\ + vector_decimal64_mul_decimal64scalar.q,\ + vector_decimal64_mul_decimal64column.q,\ vector_decimal_1.q,\ vector_decimal_10_0.q,\ vector_decimal_2.q,\ diff --git a/ql/src/gen/vectorization/ExpressionTemplates/Decimal64ColumnArithmeticDecimal64Scalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/Decimal64ColumnArithmeticDecimal64Scalar.txt index 35d9f06763..812fd203d3 100644 --- a/ql/src/gen/vectorization/ExpressionTemplates/Decimal64ColumnArithmeticDecimal64Scalar.txt +++ b/ql/src/gen/vectorization/ExpressionTemplates/Decimal64ColumnArithmeticDecimal64Scalar.txt @@ -30,15 +30,15 @@ import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.ql.metadata.HiveException; /** - * Generated from template ColumnArithmeticScalar.txt, which covers decimal64 arithmetic + * Generated from template Decimal64ColumnArithmeticDecimal64Scalar.txt, which covers decimal64 arithmetic * expressions between a column and a scalar. */ public class extends VectorExpression { private static final long serialVersionUID = 1L; - private final int colNum; - private final long value; + protected final int colNum; + protected final long value; public (int colNum, long value, int outputColumnNum) { super(outputColumnNum); @@ -212,4 +212,4 @@ public class extends VectorExpression { VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); } -} \ No newline at end of file +} diff --git a/ql/src/gen/vectorization/ExpressionTemplates/Decimal64ColumnArithmeticDecimal64ScalarUnscaled.txt b/ql/src/gen/vectorization/ExpressionTemplates/Decimal64ColumnArithmeticDecimal64ScalarUnscaled.txt new file mode 100644 index 0000000000..3e84ecdcb1 --- /dev/null +++ b/ql/src/gen/vectorization/ExpressionTemplates/Decimal64ColumnArithmeticDecimal64ScalarUnscaled.txt @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; + +/** + * Generated from template Decimal64ColumnArithmeticDecimal64ScalarUnscaled.txt, which covers decimal64 arithmetic + * expressions between a column and a scalar unscaled version. + */ +public class extends { + + public (int colNum, long value, int outputColumnNum) { + super(colNum, value, outputColumnNum); + } + + public () { + super(); + } + + @Override + public String vectorExpressionParameters() { + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) inputTypeInfos[1]; + HiveDecimalWritable writable = new HiveDecimalWritable(); + writable.deserialize64(value, decimalTypeInfo.scale()); + return getColumnParamString(0, colNum) + ", decimal64Val " + value + + ", decimalVal " + writable.toString(); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.DECIMAL_64, + VectorExpressionDescriptor.ArgumentType.DECIMAL_64) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR) + .setUnscaled(true).build(); + } + +} diff --git a/ql/src/gen/vectorization/ExpressionTemplates/Decimal64ScalarArithmeticDecimal64Column.txt b/ql/src/gen/vectorization/ExpressionTemplates/Decimal64ScalarArithmeticDecimal64Column.txt index ca5b4bc9b4..90d1d372fb 100644 --- a/ql/src/gen/vectorization/ExpressionTemplates/Decimal64ScalarArithmeticDecimal64Column.txt +++ b/ql/src/gen/vectorization/ExpressionTemplates/Decimal64ScalarArithmeticDecimal64Column.txt @@ -38,8 +38,8 @@ public class extends VectorExpression { private static final long serialVersionUID = 1L; - private int colNum; - private long value; + protected final int colNum; + protected final long value; public (long value, int colNum, int outputColumnNum) { super(outputColumnNum); diff --git a/ql/src/gen/vectorization/ExpressionTemplates/Decimal64ScalarArithmeticDecimal64ColumnUnscaled.txt b/ql/src/gen/vectorization/ExpressionTemplates/Decimal64ScalarArithmeticDecimal64ColumnUnscaled.txt new file mode 100644 index 0000000000..c59761db20 --- /dev/null +++ b/ql/src/gen/vectorization/ExpressionTemplates/Decimal64ScalarArithmeticDecimal64ColumnUnscaled.txt @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import java.util.Arrays; + +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; + +/** + * Generated from template Decimal64ScalarArithmeticDecimal64ColumnUnscaled.txt. + * Implements a vectorized arithmetic operator with a scalar on the left and a + * column vector on the right unscaled. The result is output to an output column vector. + */ +public class extends { + + public (long value, int colNum, int outputColumnNum) { + super(value, colNum, outputColumnNum); + } + + public () { + super(); + } + + public String vectorExpressionParameters() { + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) inputTypeInfos[0]; + HiveDecimalWritable writable = new HiveDecimalWritable(); + writable.deserialize64(value, decimalTypeInfo.scale()); + return "decimal64Val " + value + ", decimalVal " + writable.toString() + + ", " + getColumnParamString(1, colNum); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.DECIMAL_64, + VectorExpressionDescriptor.ArgumentType.DECIMAL_64) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.SCALAR, + VectorExpressionDescriptor.InputExpressionType.COLUMN) + .setUnscaled(true).build(); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java index e8e0f64b5d..92642c1e61 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java @@ -208,6 +208,7 @@ public int getValue() { private Mode mode = Mode.PROJECTION; ArgumentType [] argTypes = new ArgumentType[MAX_NUM_ARGUMENTS]; InputExpressionType [] exprTypes = new InputExpressionType[MAX_NUM_ARGUMENTS]; + private boolean unscaled; private int argCount = 0; public Builder() { @@ -263,8 +264,13 @@ public Builder setInputExpressionType(int index, InputExpressionType type) { return this; } + public Builder setUnscaled(boolean unscaled) { + this.unscaled = unscaled; + return this; + } + public Descriptor build() { - return new Descriptor(mode, argCount, argTypes, exprTypes); + return new Descriptor(mode, argCount, argTypes, exprTypes, unscaled); } } @@ -278,6 +284,9 @@ public boolean matches(Descriptor other) { if (!mode.equals(other.mode) || (argCount != other.argCount) ) { return false; } + if (unscaled != other.unscaled) { + return false; + } for (int i = 0; i < argCount; i++) { if (!argTypes[i].isSameTypeOrFamily(other.argTypes[i])) { return false; @@ -293,12 +302,14 @@ public boolean matches(Descriptor other) { private final ArgumentType [] argTypes; private final InputExpressionType [] exprTypes; private final int argCount; + private final boolean unscaled; - private Descriptor(Mode mode, int argCount, ArgumentType[] argTypes, InputExpressionType[] exprTypes) { + private Descriptor(Mode mode, int argCount, ArgumentType[] argTypes, InputExpressionType[] exprTypes, boolean unscaled) { this.mode = mode; this.argTypes = argTypes.clone(); this.exprTypes = exprTypes.clone(); this.argCount = argCount; + this.unscaled = unscaled; } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index ce6adc4a6b..bac21beac7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -826,7 +826,7 @@ public void freeMarkedScratchColumns() { private VectorExpression getFilterOnBooleanColumnExpression(ExprNodeColumnDesc exprDesc, int columnNum) throws HiveException { - VectorExpression expr; + final VectorExpression expr; // Evaluate the column as a boolean, converting if necessary. TypeInfo typeInfo = exprDesc.getTypeInfo(); @@ -1776,6 +1776,7 @@ private VectorExpression getDecimal64VectorExpressionForUdf(GenericUDF genericUd final boolean isReturnDecimal64 = checkTypeInfoForDecimal64(returnTypeInfo); final DataTypePhysicalVariation returnDataTypePhysicalVariation; + final boolean dontRescaleArguments = (genericUdf instanceof GenericUDFOPMultiply); if (isReturnDecimal64) { DecimalTypeInfo returnDecimalTypeInfo = (DecimalTypeInfo) returnTypeInfo; if (!isDecimal64ScaleEstablished) { @@ -1788,7 +1789,7 @@ private VectorExpression getDecimal64VectorExpressionForUdf(GenericUDF genericUd if((leftType.precision() + returnDecimalTypeInfo.getScale()) > 18) { return null; } - } else if (returnDecimalTypeInfo.getScale() != decimal64ColumnScale) { + } else if (returnDecimalTypeInfo.getScale() != decimal64ColumnScale && !dontRescaleArguments) { return null; } returnDataTypePhysicalVariation = DataTypePhysicalVariation.DECIMAL_64; @@ -1802,6 +1803,9 @@ private VectorExpression getDecimal64VectorExpressionForUdf(GenericUDF genericUd returnDataTypePhysicalVariation = DataTypePhysicalVariation.NONE; } + if(dontRescaleArguments) { + builder.setUnscaled(true); + } VectorExpressionDescriptor.Descriptor descriptor = builder.build(); Class vectorClass = this.vMap.getVectorExpressionClass(udfClass, descriptor, useCheckedVectorExpressions); @@ -1814,13 +1818,15 @@ private VectorExpression getDecimal64VectorExpressionForUdf(GenericUDF genericUd return createDecimal64VectorExpression( vectorClass, childExprs, childrenMode, isDecimal64ScaleEstablished, decimal64ColumnScale, - returnTypeInfo, returnDataTypePhysicalVariation); + returnTypeInfo, returnDataTypePhysicalVariation, dontRescaleArguments); } + @SuppressWarnings("null") private VectorExpression createDecimal64VectorExpression(Class vectorClass, List childExprs, VectorExpressionDescriptor.Mode childrenMode, boolean isDecimal64ScaleEstablished, int decimal64ColumnScale, - TypeInfo returnTypeInfo, DataTypePhysicalVariation returnDataTypePhysicalVariation) + TypeInfo returnTypeInfo, DataTypePhysicalVariation returnDataTypePhysicalVariation, + boolean dontRescaleArguments) throws HiveException { final int numChildren = childExprs.size(); @@ -1870,9 +1876,11 @@ private VectorExpression createDecimal64VectorExpression(Class vectorClass, // For now, bail out on decimal constants with larger scale than column scale. return null; } - final long decimal64Scalar = - new HiveDecimalWritable(hiveDecimal).serialize64(decimal64ColumnScale); - arguments[i] = decimal64Scalar; + if (dontRescaleArguments) { + arguments[i] = new HiveDecimalWritable(hiveDecimal).serialize64(hiveDecimal.scale()); + } else { + arguments[i] = new HiveDecimalWritable(hiveDecimal).serialize64(decimal64ColumnScale); + } } else { Object scalarValue = getVectorTypeScalarValue(constDesc); arguments[i] = @@ -2770,7 +2778,8 @@ private VectorExpression getInExpression(List childExpr, cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, /* isDecimal64ScaleEstablished */ true, /* decimal64ColumnScale */ scale, - returnType, DataTypePhysicalVariation.NONE); + returnType, DataTypePhysicalVariation.NONE, + /* dontRescaleArguments */ false); if (expr != null) { long[] inVals = new long[childrenForInList.size()]; for (int i = 0; i != inVals.length; i++) { @@ -3398,7 +3407,8 @@ private VectorExpression tryDecimal64Between(VectorExpressionDescriptor.Mode mod cl, childrenAfterNot, VectorExpressionDescriptor.Mode.PROJECTION, /* isDecimal64ScaleEstablished */ true, /* decimal64ColumnScale */ ((DecimalTypeInfo) colExpr.getTypeInfo()).getScale(), - returnTypeInfo, DataTypePhysicalVariation.NONE); + returnTypeInfo, DataTypePhysicalVariation.NONE, + /* dontRescaleArguments */ false); } /* Get a [NOT] BETWEEN filter or projection expression. This is treated as a special case diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPDivide.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPDivide.java index 0c0ce68311..dfee4115b2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPDivide.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPDivide.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressionsSupportDecimal64; import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColDivideLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColDivideLongScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.LongScalarDivideLongColumn; @@ -50,6 +51,7 @@ DecimalColDivideDecimalColumn.class, DecimalColDivideDecimalScalar.class, DecimalScalarDivideDecimalColumn.class, Decimal64ColDivideDecimal64Scalar.class, Decimal64ColDivideDecimal64Column.class}) +@VectorizedExpressionsSupportDecimal64() public class GenericUDFOPDivide extends GenericUDFBaseNumeric { public GenericUDFOPDivide() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMultiply.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMultiply.java index 616641d2f0..f2859eefab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMultiply.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMultiply.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressionsSupportDecimal64; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; @@ -46,7 +47,9 @@ DoubleScalarMultiplyLongColumn.class, DoubleScalarMultiplyDoubleColumn.class, DoubleScalarMultiplyLongColumnChecked.class, DoubleScalarMultiplyDoubleColumnChecked.class, DecimalColMultiplyDecimalColumn.class, DecimalColMultiplyDecimalScalar.class, - DecimalScalarMultiplyDecimalColumn.class}) + DecimalScalarMultiplyDecimalColumn.class, Decimal64ColMultiplyDecimal64ScalarUnscaled.class, + Decimal64ColMultiplyDecimal64Column.class, Decimal64ScalarMultiplyDecimal64ColumnUnscaled.class}) +@VectorizedExpressionsSupportDecimal64() public class GenericUDFOPMultiply extends GenericUDFBaseNumeric { public GenericUDFOPMultiply() { diff --git a/ql/src/test/queries/clientpositive/vector_decimal64_mul_decimal64column.q b/ql/src/test/queries/clientpositive/vector_decimal64_mul_decimal64column.q new file mode 100644 index 0000000000..d660d39db5 --- /dev/null +++ b/ql/src/test/queries/clientpositive/vector_decimal64_mul_decimal64column.q @@ -0,0 +1,6 @@ +create external table vector_decimal64_mul_decimal64column(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE; +LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_decimal64column; +create table vector_decimal64_mul_decimal64column_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) stored as ORC; +insert into table vector_decimal64_mul_decimal64column_tmp select * from vector_decimal64_mul_decimal64column; +explain vectorization detail select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_decimal64column_tmp; +select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_decimal64column_tmp; diff --git a/ql/src/test/queries/clientpositive/vector_decimal64_mul_decimal64scalar.q b/ql/src/test/queries/clientpositive/vector_decimal64_mul_decimal64scalar.q new file mode 100644 index 0000000000..9c88f584c9 --- /dev/null +++ b/ql/src/test/queries/clientpositive/vector_decimal64_mul_decimal64scalar.q @@ -0,0 +1,6 @@ +create external table vector_decimal64_mul_decimal64scalar(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE; +LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_decimal64scalar; +create table vector_decimal64_mul_decimal64scalar_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) stored as ORC; +insert into table vector_decimal64_mul_decimal64scalar_tmp select * from vector_decimal64_mul_decimal64scalar; +explain vectorization detail select sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)*2.2) from vector_decimal64_mul_decimal64scalar_tmp; +select sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)*2.2) from vector_decimal64_mul_decimal64scalar_tmp; diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal64_mul_decimal64column.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal64_mul_decimal64column.q.out new file mode 100644 index 0000000000..ae4ac0288f --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/vector_decimal64_mul_decimal64column.q.out @@ -0,0 +1,170 @@ +PREHOOK: query: create external table vector_decimal64_mul_decimal64column(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vector_decimal64_mul_decimal64column +POSTHOOK: query: create external table vector_decimal64_mul_decimal64column(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vector_decimal64_mul_decimal64column +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_decimal64column +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@vector_decimal64_mul_decimal64column +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_decimal64column +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@vector_decimal64_mul_decimal64column +PREHOOK: query: create table vector_decimal64_mul_decimal64column_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) stored as ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vector_decimal64_mul_decimal64column_tmp +POSTHOOK: query: create table vector_decimal64_mul_decimal64column_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) stored as ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vector_decimal64_mul_decimal64column_tmp +PREHOOK: query: insert into table vector_decimal64_mul_decimal64column_tmp select * from vector_decimal64_mul_decimal64column +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64column +PREHOOK: Output: default@vector_decimal64_mul_decimal64column_tmp +POSTHOOK: query: insert into table vector_decimal64_mul_decimal64column_tmp select * from vector_decimal64_mul_decimal64column +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64column +POSTHOOK: Output: default@vector_decimal64_mul_decimal64column_tmp +POSTHOOK: Lineage: vector_decimal64_mul_decimal64column_tmp.ss_ext_discount_amt SIMPLE [(vector_decimal64_mul_decimal64column)vector_decimal64_mul_decimal64column.FieldSchema(name:ss_ext_discount_amt, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64column_tmp.ss_ext_list_price SIMPLE [(vector_decimal64_mul_decimal64column)vector_decimal64_mul_decimal64column.FieldSchema(name:ss_ext_list_price, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64column_tmp.ss_ext_sales_price SIMPLE [(vector_decimal64_mul_decimal64column)vector_decimal64_mul_decimal64column.FieldSchema(name:ss_ext_sales_price, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64column_tmp.ss_ext_wholesale_cost SIMPLE [(vector_decimal64_mul_decimal64column)vector_decimal64_mul_decimal64column.FieldSchema(name:ss_ext_wholesale_cost, type:decimal(7,2), comment:null), ] +PREHOOK: query: explain vectorization detail select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_decimal64column_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64column_tmp +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_decimal64column_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64column_tmp +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: vector_decimal64_mul_decimal64column_tmp + Statistics: Num rows: 1000 Data size: 224000 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:ss_ext_list_price:decimal(7,2)/DECIMAL_64, 1:ss_ext_wholesale_cost:decimal(7,2)/DECIMAL_64, 2:ss_ext_discount_amt:decimal(7,2)/DECIMAL_64, 3:ss_ext_sales_price:decimal(7,2)/DECIMAL_64, 4:ROW__ID:struct] + Select Operator + expressions: (ss_ext_list_price * ss_ext_discount_amt) (type: decimal(15,4)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [5] + selectExpressions: Decimal64ColMultiplyDecimal64Column(col 0:decimal(7,2)/DECIMAL_64, col 2:decimal(7,2)/DECIMAL_64) -> 5:decimal(15,4)/DECIMAL_64 + Statistics: Num rows: 1000 Data size: 224000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal64ToDecimal(col 5:decimal(15,4)/DECIMAL_64) -> decimal(25,4) + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 0:decimal(25,4) + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(25,4)) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 4 + includeColumns: [0, 2] + dataColumns: ss_ext_list_price:decimal(7,2)/DECIMAL_64, ss_ext_wholesale_cost:decimal(7,2)/DECIMAL_64, ss_ext_discount_amt:decimal(7,2)/DECIMAL_64, ss_ext_sales_price:decimal(7,2)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [decimal(15,4)/DECIMAL_64] + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: + reduceColumnSortOrder: + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + dataColumns: VALUE._col0:decimal(25,4) + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal(col 0:decimal(25,4)) -> decimal(25,4) + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumnNums: [0] + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_decimal64column_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64column_tmp +#### A masked pattern was here #### +POSTHOOK: query: select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_decimal64column_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64column_tmp +#### A masked pattern was here #### +984383500.0000 diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal64_mul_decimal64scalar.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal64_mul_decimal64scalar.q.out new file mode 100644 index 0000000000..3473425667 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/vector_decimal64_mul_decimal64scalar.q.out @@ -0,0 +1,171 @@ +PREHOOK: query: create external table vector_decimal64_mul_decimal64scalar(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vector_decimal64_mul_decimal64scalar +POSTHOOK: query: create external table vector_decimal64_mul_decimal64scalar(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vector_decimal64_mul_decimal64scalar +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_decimal64scalar +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@vector_decimal64_mul_decimal64scalar +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_decimal64scalar +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@vector_decimal64_mul_decimal64scalar +PREHOOK: query: create table vector_decimal64_mul_decimal64scalar_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) stored as ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vector_decimal64_mul_decimal64scalar_tmp +POSTHOOK: query: create table vector_decimal64_mul_decimal64scalar_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) stored as ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vector_decimal64_mul_decimal64scalar_tmp +PREHOOK: query: insert into table vector_decimal64_mul_decimal64scalar_tmp select * from vector_decimal64_mul_decimal64scalar +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64scalar +PREHOOK: Output: default@vector_decimal64_mul_decimal64scalar_tmp +POSTHOOK: query: insert into table vector_decimal64_mul_decimal64scalar_tmp select * from vector_decimal64_mul_decimal64scalar +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64scalar +POSTHOOK: Output: default@vector_decimal64_mul_decimal64scalar_tmp +POSTHOOK: Lineage: vector_decimal64_mul_decimal64scalar_tmp.ss_ext_discount_amt SIMPLE [(vector_decimal64_mul_decimal64scalar)vector_decimal64_mul_decimal64scalar.FieldSchema(name:ss_ext_discount_amt, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64scalar_tmp.ss_ext_list_price SIMPLE [(vector_decimal64_mul_decimal64scalar)vector_decimal64_mul_decimal64scalar.FieldSchema(name:ss_ext_list_price, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64scalar_tmp.ss_ext_sales_price SIMPLE [(vector_decimal64_mul_decimal64scalar)vector_decimal64_mul_decimal64scalar.FieldSchema(name:ss_ext_sales_price, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64scalar_tmp.ss_ext_wholesale_cost SIMPLE [(vector_decimal64_mul_decimal64scalar)vector_decimal64_mul_decimal64scalar.FieldSchema(name:ss_ext_wholesale_cost, type:decimal(7,2), comment:null), ] +PREHOOK: query: explain vectorization detail select sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)*2.2) from vector_decimal64_mul_decimal64scalar_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64scalar_tmp +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail select sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)*2.2) from vector_decimal64_mul_decimal64scalar_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64scalar_tmp +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: vector_decimal64_mul_decimal64scalar_tmp + Statistics: Num rows: 1000 Data size: 448000 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:ss_ext_list_price:decimal(7,2)/DECIMAL_64, 1:ss_ext_wholesale_cost:decimal(7,2)/DECIMAL_64, 2:ss_ext_discount_amt:decimal(7,2)/DECIMAL_64, 3:ss_ext_sales_price:decimal(7,2)/DECIMAL_64, 4:ROW__ID:struct] + Select Operator + expressions: ((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) * 2.2) (type: decimal(13,3)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [8] + selectExpressions: Decimal64ColMultiplyDecimal64ScalarUnscaled(col 7:decimal(10,2)/DECIMAL_64, decimal64Val 22, decimalVal 2.2)(children: Decimal64ColAddDecimal64Column(col 6:decimal(9,2)/DECIMAL_64, col 3:decimal(7,2)/DECIMAL_64)(children: Decimal64ColSubtractDecimal64Column(col 5:decimal(8,2)/DECIMAL_64, col 2:decimal(7,2)/DECIMAL_64)(children: Decimal64ColSubtractDecimal64Column(col 0:decimal(7,2)/DECIMAL_64, col 1:decimal(7,2)/DECIMAL_64) -> 5:decimal(8,2)/DECIMAL_64) -> 6:decimal(9,2)/DECIMAL_64) -> 7:decimal(10,2)/DECIMAL_64) -> 8:decimal(13,3)/DECIMAL_64 + Statistics: Num rows: 1000 Data size: 448000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal64ToDecimal(col 8:decimal(13,3)/DECIMAL_64) -> decimal(23,3) + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 0:decimal(23,3) + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(23,3)) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 4 + includeColumns: [0, 1, 2, 3] + dataColumns: ss_ext_list_price:decimal(7,2)/DECIMAL_64, ss_ext_wholesale_cost:decimal(7,2)/DECIMAL_64, ss_ext_discount_amt:decimal(7,2)/DECIMAL_64, ss_ext_sales_price:decimal(7,2)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [decimal(8,2)/DECIMAL_64, decimal(9,2)/DECIMAL_64, decimal(10,2)/DECIMAL_64, decimal(13,3)/DECIMAL_64] + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: + reduceColumnSortOrder: + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + dataColumns: VALUE._col0:decimal(23,3) + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal(col 0:decimal(23,3)) -> decimal(23,3) + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumnNums: [0] + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)*2.2) from vector_decimal64_mul_decimal64scalar_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64scalar_tmp +#### A masked pattern was here #### +POSTHOOK: query: select sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)*2.2) from vector_decimal64_mul_decimal64scalar_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64scalar_tmp +#### A masked pattern was here #### +2002000.000 diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out index 468218af05..8500562893 100644 --- a/ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out @@ -258,8 +258,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [7, 11, 14, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37] - selectExpressions: DecimalColAddDecimalColumn(col 5:decimal(10,3), col 6:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 5:decimal(10,3), ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 6:decimal(7,2)) -> 7:decimal(11,3), DecimalColSubtractDecimalColumn(col 8:decimal(10,3), col 10:decimal(9,2))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 8:decimal(10,3), DecimalScalarMultiplyDecimalColumn(val 2, col 9:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 9:decimal(7,2)) -> 10:decimal(9,2)) -> 11:decimal(11,3), DecimalColDivideDecimalColumn(col 38:decimal(11,3), col 13:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 12:decimal(11,3)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,3)/DECIMAL_64, decimal64Val 2340, decimalVal 2.34) -> 12:decimal(11,3)/DECIMAL_64) -> 38:decimal(11,3), ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 13:decimal(7,2)) -> 14:decimal(21,11), DecimalColMultiplyDecimalColumn(col 15:decimal(10,3), col 39:decimal(12,6))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 15:decimal(10,3), ConvertDecimal64ToDecimal(col 16:decimal(12,6)/DECIMAL_64)(children: Decimal64ColDivideDecimal64Scalar(col 2:decimal(7,2)/DECIMAL_64, decimal64Val 340, decimalVal 3.4) -> 16:decimal(12,6)/DECIMAL_64) -> 39:decimal(12,6)) -> 17:decimal(23,9), DecimalColModuloDecimalScalar(col 18:decimal(10,3), val 10)(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 18:decimal(10,3)) -> 19:decimal(5,3), CastDecimalToLong(col 20:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 20:decimal(10,3)) -> 21:int, CastDecimalToLong(col 22:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 22:decimal(7,2)) -> 23:smallint, CastDecimalToLong(col 24:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 24:decimal(7,2)) -> 25:tinyint, CastDecimalToLong(col 26:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 26:decimal(10,3)) -> 27:bigint, CastDecimalToBoolean(col 28:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 28:decimal(10,3)) -> 29:boolean, CastDecimalToDouble(col 30:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 30:decimal(7,2)) -> 31:double, CastDecimalToFloat(col 32:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 32:decimal(10,3)) -> 33:float, CastDecimalToString(col 34:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 34:decimal(7,2)) -> 35:string, CastDecimalToTimestamp(col 36:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 36:decimal(10,3)) -> 37:timestamp + projectedOutputColumnNums: [7, 10, 13, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36] + selectExpressions: DecimalColAddDecimalColumn(col 5:decimal(10,3), col 6:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 5:decimal(10,3), ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 6:decimal(7,2)) -> 7:decimal(11,3), DecimalColSubtractDecimalColumn(col 8:decimal(10,3), col 37:decimal(9,2))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 8:decimal(10,3), ConvertDecimal64ToDecimal(col 9:decimal(9,2)/DECIMAL_64)(children: Decimal64ScalarMultiplyDecimal64ColumnUnscaled(decimal64Val 2, decimalVal 2, col 2:decimal(7,2)/DECIMAL_64) -> 9:decimal(9,2)/DECIMAL_64) -> 37:decimal(9,2)) -> 10:decimal(11,3), DecimalColDivideDecimalColumn(col 38:decimal(11,3), col 12:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 11:decimal(11,3)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,3)/DECIMAL_64, decimal64Val 2340, decimalVal 2.34) -> 11:decimal(11,3)/DECIMAL_64) -> 38:decimal(11,3), ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 12:decimal(7,2)) -> 13:decimal(21,11), DecimalColMultiplyDecimalColumn(col 14:decimal(10,3), col 39:decimal(12,6))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 14:decimal(10,3), ConvertDecimal64ToDecimal(col 15:decimal(12,6)/DECIMAL_64)(children: Decimal64ColDivideDecimal64Scalar(col 2:decimal(7,2)/DECIMAL_64, decimal64Val 340, decimalVal 3.4) -> 15:decimal(12,6)/DECIMAL_64) -> 39:decimal(12,6)) -> 16:decimal(23,9), DecimalColModuloDecimalScalar(col 17:decimal(10,3), val 10)(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 17:decimal(10,3)) -> 18:decimal(5,3), CastDecimalToLong(col 19:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 19:decimal(10,3)) -> 20:int, CastDecimalToLong(col 21:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 21:decimal(7,2)) -> 22:smallint, CastDecimalToLong(col 23:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 23:decimal(7,2)) -> 24:tinyint, CastDecimalToLong(col 25:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 25:decimal(10,3)) -> 26:bigint, CastDecimalToBoolean(col 27:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 27:decimal(10,3)) -> 28:boolean, CastDecimalToDouble(col 29:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 29:decimal(7,2)) -> 30:double, CastDecimalToFloat(col 31:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 31:decimal(10,3)) -> 32:float, CastDecimalToString(col 33:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 33:decimal(7,2)) -> 34:string, CastDecimalToTimestamp(col 35:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 35:decimal(10,3)) -> 36:timestamp Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(11,3)), _col1 (type: decimal(11,3)), _col2 (type: decimal(21,11)), _col3 (type: decimal(23,9)), _col4 (type: decimal(5,3)), _col5 (type: int), _col6 (type: smallint), _col7 (type: tinyint), _col8 (type: bigint), _col9 (type: boolean), _col10 (type: double), _col11 (type: float), _col12 (type: string), _col13 (type: timestamp) @@ -267,7 +267,7 @@ STAGE PLANS: sort order: ++++++++++++++ Reduce Sink Vectorization: className: VectorReduceSinkObjectHashOperator - keyColumns: 7:decimal(11,3), 11:decimal(11,3), 14:decimal(21,11), 17:decimal(23,9), 19:decimal(5,3), 21:int, 23:smallint, 25:tinyint, 27:bigint, 29:boolean, 31:double, 33:float, 35:string, 37:timestamp + keyColumns: 7:decimal(11,3), 10:decimal(11,3), 13:decimal(21,11), 16:decimal(23,9), 18:decimal(5,3), 20:int, 22:smallint, 24:tinyint, 26:bigint, 28:boolean, 30:double, 32:float, 34:string, 36:timestamp native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE @@ -288,7 +288,7 @@ STAGE PLANS: includeColumns: [0, 1, 2] dataColumns: cdouble:double, cdecimal1:decimal(10,3)/DECIMAL_64, cdecimal2:decimal(7,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(10,3), decimal(10,3), decimal(7,2), decimal(11,3), decimal(10,3), decimal(7,2), decimal(9,2), decimal(11,3), decimal(11,3)/DECIMAL_64, decimal(7,2), decimal(21,11), decimal(10,3), decimal(12,6)/DECIMAL_64, decimal(23,9), decimal(10,3), decimal(5,3), decimal(10,3), bigint, decimal(7,2), bigint, decimal(7,2), bigint, decimal(10,3), bigint, decimal(10,3), bigint, decimal(7,2), double, decimal(10,3), double, decimal(7,2), string, decimal(10,3), timestamp, decimal(11,3), decimal(12,6)] + scratchColumnTypeNames: [decimal(10,3), decimal(10,3), decimal(7,2), decimal(11,3), decimal(10,3), decimal(9,2)/DECIMAL_64, decimal(11,3), decimal(11,3)/DECIMAL_64, decimal(7,2), decimal(21,11), decimal(10,3), decimal(12,6)/DECIMAL_64, decimal(23,9), decimal(10,3), decimal(5,3), decimal(10,3), bigint, decimal(7,2), bigint, decimal(7,2), bigint, decimal(10,3), bigint, decimal(10,3), bigint, decimal(7,2), double, decimal(10,3), double, decimal(7,2), string, decimal(10,3), timestamp, decimal(9,2), decimal(11,3), decimal(12,6)] Reducer 2 Execution mode: vectorized, llap Reduce Vectorization: diff --git a/ql/src/test/results/clientpositive/vector_decimal64_mul_decimal64column.q.out b/ql/src/test/results/clientpositive/vector_decimal64_mul_decimal64column.q.out new file mode 100644 index 0000000000..a4cbd23d1b --- /dev/null +++ b/ql/src/test/results/clientpositive/vector_decimal64_mul_decimal64column.q.out @@ -0,0 +1,142 @@ +PREHOOK: query: create external table vector_decimal64_mul_decimal64column(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vector_decimal64_mul_decimal64column +POSTHOOK: query: create external table vector_decimal64_mul_decimal64column(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vector_decimal64_mul_decimal64column +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_decimal64column +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@vector_decimal64_mul_decimal64column +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_decimal64column +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@vector_decimal64_mul_decimal64column +PREHOOK: query: create table vector_decimal64_mul_decimal64column_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) stored as ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vector_decimal64_mul_decimal64column_tmp +POSTHOOK: query: create table vector_decimal64_mul_decimal64column_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) stored as ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vector_decimal64_mul_decimal64column_tmp +PREHOOK: query: insert into table vector_decimal64_mul_decimal64column_tmp select * from vector_decimal64_mul_decimal64column +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64column +PREHOOK: Output: default@vector_decimal64_mul_decimal64column_tmp +POSTHOOK: query: insert into table vector_decimal64_mul_decimal64column_tmp select * from vector_decimal64_mul_decimal64column +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64column +POSTHOOK: Output: default@vector_decimal64_mul_decimal64column_tmp +POSTHOOK: Lineage: vector_decimal64_mul_decimal64column_tmp.ss_ext_discount_amt SIMPLE [(vector_decimal64_mul_decimal64column)vector_decimal64_mul_decimal64column.FieldSchema(name:ss_ext_discount_amt, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64column_tmp.ss_ext_list_price SIMPLE [(vector_decimal64_mul_decimal64column)vector_decimal64_mul_decimal64column.FieldSchema(name:ss_ext_list_price, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64column_tmp.ss_ext_sales_price SIMPLE [(vector_decimal64_mul_decimal64column)vector_decimal64_mul_decimal64column.FieldSchema(name:ss_ext_sales_price, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64column_tmp.ss_ext_wholesale_cost SIMPLE [(vector_decimal64_mul_decimal64column)vector_decimal64_mul_decimal64column.FieldSchema(name:ss_ext_wholesale_cost, type:decimal(7,2), comment:null), ] +PREHOOK: query: explain vectorization detail select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_decimal64column_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64column_tmp +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_decimal64column_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64column_tmp +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: vector_decimal64_mul_decimal64column_tmp + Statistics: Num rows: 1000 Data size: 224000 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:ss_ext_list_price:decimal(7,2)/DECIMAL_64, 1:ss_ext_wholesale_cost:decimal(7,2)/DECIMAL_64, 2:ss_ext_discount_amt:decimal(7,2)/DECIMAL_64, 3:ss_ext_sales_price:decimal(7,2)/DECIMAL_64, 4:ROW__ID:struct] + Select Operator + expressions: (ss_ext_list_price * ss_ext_discount_amt) (type: decimal(15,4)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [5] + selectExpressions: Decimal64ColMultiplyDecimal64Column(col 0:decimal(7,2)/DECIMAL_64, col 2:decimal(7,2)/DECIMAL_64) -> 5:decimal(15,4)/DECIMAL_64 + Statistics: Num rows: 1000 Data size: 224000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal64ToDecimal(col 5:decimal(15,4)/DECIMAL_64) -> decimal(25,4) + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkOperator + native: false + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(25,4)) + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 4 + includeColumns: [0, 2] + dataColumns: ss_ext_list_price:decimal(7,2)/DECIMAL_64, ss_ext_wholesale_cost:decimal(7,2)/DECIMAL_64, ss_ext_discount_amt:decimal(7,2)/DECIMAL_64, ss_ext_sales_price:decimal(7,2)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [decimal(15,4)/DECIMAL_64] + Reduce Vectorization: + enabled: false + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true + enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_decimal64column_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64column_tmp +#### A masked pattern was here #### +POSTHOOK: query: select sum(ss_ext_list_price*ss_ext_discount_amt) from vector_decimal64_mul_decimal64column_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64column_tmp +#### A masked pattern was here #### +984383500.0000 diff --git a/ql/src/test/results/clientpositive/vector_decimal64_mul_decimal64scalar.q.out b/ql/src/test/results/clientpositive/vector_decimal64_mul_decimal64scalar.q.out new file mode 100644 index 0000000000..a8f0daca1e --- /dev/null +++ b/ql/src/test/results/clientpositive/vector_decimal64_mul_decimal64scalar.q.out @@ -0,0 +1,143 @@ +PREHOOK: query: create external table vector_decimal64_mul_decimal64scalar(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vector_decimal64_mul_decimal64scalar +POSTHOOK: query: create external table vector_decimal64_mul_decimal64scalar(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vector_decimal64_mul_decimal64scalar +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_decimal64scalar +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@vector_decimal64_mul_decimal64scalar +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/decimal64table.csv' OVERWRITE INTO TABLE vector_decimal64_mul_decimal64scalar +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@vector_decimal64_mul_decimal64scalar +PREHOOK: query: create table vector_decimal64_mul_decimal64scalar_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) stored as ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vector_decimal64_mul_decimal64scalar_tmp +POSTHOOK: query: create table vector_decimal64_mul_decimal64scalar_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(7,2), ss_ext_discount_amt decimal(7,2), ss_ext_sales_price decimal(7,2)) stored as ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vector_decimal64_mul_decimal64scalar_tmp +PREHOOK: query: insert into table vector_decimal64_mul_decimal64scalar_tmp select * from vector_decimal64_mul_decimal64scalar +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64scalar +PREHOOK: Output: default@vector_decimal64_mul_decimal64scalar_tmp +POSTHOOK: query: insert into table vector_decimal64_mul_decimal64scalar_tmp select * from vector_decimal64_mul_decimal64scalar +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64scalar +POSTHOOK: Output: default@vector_decimal64_mul_decimal64scalar_tmp +POSTHOOK: Lineage: vector_decimal64_mul_decimal64scalar_tmp.ss_ext_discount_amt SIMPLE [(vector_decimal64_mul_decimal64scalar)vector_decimal64_mul_decimal64scalar.FieldSchema(name:ss_ext_discount_amt, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64scalar_tmp.ss_ext_list_price SIMPLE [(vector_decimal64_mul_decimal64scalar)vector_decimal64_mul_decimal64scalar.FieldSchema(name:ss_ext_list_price, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64scalar_tmp.ss_ext_sales_price SIMPLE [(vector_decimal64_mul_decimal64scalar)vector_decimal64_mul_decimal64scalar.FieldSchema(name:ss_ext_sales_price, type:decimal(7,2), comment:null), ] +POSTHOOK: Lineage: vector_decimal64_mul_decimal64scalar_tmp.ss_ext_wholesale_cost SIMPLE [(vector_decimal64_mul_decimal64scalar)vector_decimal64_mul_decimal64scalar.FieldSchema(name:ss_ext_wholesale_cost, type:decimal(7,2), comment:null), ] +PREHOOK: query: explain vectorization detail select sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)*2.2) from vector_decimal64_mul_decimal64scalar_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64scalar_tmp +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail select sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)*2.2) from vector_decimal64_mul_decimal64scalar_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64scalar_tmp +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: vector_decimal64_mul_decimal64scalar_tmp + Statistics: Num rows: 1000 Data size: 448000 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:ss_ext_list_price:decimal(7,2)/DECIMAL_64, 1:ss_ext_wholesale_cost:decimal(7,2)/DECIMAL_64, 2:ss_ext_discount_amt:decimal(7,2)/DECIMAL_64, 3:ss_ext_sales_price:decimal(7,2)/DECIMAL_64, 4:ROW__ID:struct] + Select Operator + expressions: ((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) * 2.2) (type: decimal(13,3)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [8] + selectExpressions: Decimal64ColMultiplyDecimal64ScalarUnscaled(col 7:decimal(10,2)/DECIMAL_64, decimal64Val 22, decimalVal 2.2)(children: Decimal64ColAddDecimal64Column(col 6:decimal(9,2)/DECIMAL_64, col 3:decimal(7,2)/DECIMAL_64)(children: Decimal64ColSubtractDecimal64Column(col 5:decimal(8,2)/DECIMAL_64, col 2:decimal(7,2)/DECIMAL_64)(children: Decimal64ColSubtractDecimal64Column(col 0:decimal(7,2)/DECIMAL_64, col 1:decimal(7,2)/DECIMAL_64) -> 5:decimal(8,2)/DECIMAL_64) -> 6:decimal(9,2)/DECIMAL_64) -> 7:decimal(10,2)/DECIMAL_64) -> 8:decimal(13,3)/DECIMAL_64 + Statistics: Num rows: 1000 Data size: 448000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal64ToDecimal(col 8:decimal(13,3)/DECIMAL_64) -> decimal(23,3) + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkOperator + native: false + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(23,3)) + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 4 + includeColumns: [0, 1, 2, 3] + dataColumns: ss_ext_list_price:decimal(7,2)/DECIMAL_64, ss_ext_wholesale_cost:decimal(7,2)/DECIMAL_64, ss_ext_discount_amt:decimal(7,2)/DECIMAL_64, ss_ext_sales_price:decimal(7,2)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [decimal(8,2)/DECIMAL_64, decimal(9,2)/DECIMAL_64, decimal(10,2)/DECIMAL_64, decimal(13,3)/DECIMAL_64] + Reduce Vectorization: + enabled: false + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true + enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)*2.2) from vector_decimal64_mul_decimal64scalar_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_mul_decimal64scalar_tmp +#### A masked pattern was here #### +POSTHOOK: query: select sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)*2.2) from vector_decimal64_mul_decimal64scalar_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_mul_decimal64scalar_tmp +#### A masked pattern was here #### +2002000.000 diff --git a/ql/src/test/results/clientpositive/vector_decimal_expressions.q.out b/ql/src/test/results/clientpositive/vector_decimal_expressions.q.out index 5b02fa133c..0cfbb6f68e 100644 --- a/ql/src/test/results/clientpositive/vector_decimal_expressions.q.out +++ b/ql/src/test/results/clientpositive/vector_decimal_expressions.q.out @@ -224,8 +224,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [7, 11, 14, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37] - selectExpressions: DecimalColAddDecimalColumn(col 5:decimal(10,3), col 6:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 5:decimal(10,3), ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 6:decimal(7,2)) -> 7:decimal(11,3), DecimalColSubtractDecimalColumn(col 8:decimal(10,3), col 10:decimal(9,2))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 8:decimal(10,3), DecimalScalarMultiplyDecimalColumn(val 2, col 9:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 9:decimal(7,2)) -> 10:decimal(9,2)) -> 11:decimal(11,3), DecimalColDivideDecimalColumn(col 38:decimal(11,3), col 13:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 12:decimal(11,3)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,3)/DECIMAL_64, decimal64Val 2340, decimalVal 2.34) -> 12:decimal(11,3)/DECIMAL_64) -> 38:decimal(11,3), ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 13:decimal(7,2)) -> 14:decimal(21,11), DecimalColMultiplyDecimalColumn(col 15:decimal(10,3), col 39:decimal(12,6))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 15:decimal(10,3), ConvertDecimal64ToDecimal(col 16:decimal(12,6)/DECIMAL_64)(children: Decimal64ColDivideDecimal64Scalar(col 2:decimal(7,2)/DECIMAL_64, decimal64Val 340, decimalVal 3.4) -> 16:decimal(12,6)/DECIMAL_64) -> 39:decimal(12,6)) -> 17:decimal(23,9), DecimalColModuloDecimalScalar(col 18:decimal(10,3), val 10)(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 18:decimal(10,3)) -> 19:decimal(5,3), CastDecimalToLong(col 20:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 20:decimal(10,3)) -> 21:int, CastDecimalToLong(col 22:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 22:decimal(7,2)) -> 23:smallint, CastDecimalToLong(col 24:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 24:decimal(7,2)) -> 25:tinyint, CastDecimalToLong(col 26:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 26:decimal(10,3)) -> 27:bigint, CastDecimalToBoolean(col 28:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 28:decimal(10,3)) -> 29:boolean, CastDecimalToDouble(col 30:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 30:decimal(7,2)) -> 31:double, CastDecimalToFloat(col 32:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 32:decimal(10,3)) -> 33:float, CastDecimalToString(col 34:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 34:decimal(7,2)) -> 35:string, CastDecimalToTimestamp(col 36:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 36:decimal(10,3)) -> 37:timestamp + projectedOutputColumnNums: [7, 10, 13, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36] + selectExpressions: DecimalColAddDecimalColumn(col 5:decimal(10,3), col 6:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 5:decimal(10,3), ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 6:decimal(7,2)) -> 7:decimal(11,3), DecimalColSubtractDecimalColumn(col 8:decimal(10,3), col 37:decimal(9,2))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 8:decimal(10,3), ConvertDecimal64ToDecimal(col 9:decimal(9,2)/DECIMAL_64)(children: Decimal64ScalarMultiplyDecimal64ColumnUnscaled(decimal64Val 2, decimalVal 2, col 2:decimal(7,2)/DECIMAL_64) -> 9:decimal(9,2)/DECIMAL_64) -> 37:decimal(9,2)) -> 10:decimal(11,3), DecimalColDivideDecimalColumn(col 38:decimal(11,3), col 12:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 11:decimal(11,3)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,3)/DECIMAL_64, decimal64Val 2340, decimalVal 2.34) -> 11:decimal(11,3)/DECIMAL_64) -> 38:decimal(11,3), ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 12:decimal(7,2)) -> 13:decimal(21,11), DecimalColMultiplyDecimalColumn(col 14:decimal(10,3), col 39:decimal(12,6))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 14:decimal(10,3), ConvertDecimal64ToDecimal(col 15:decimal(12,6)/DECIMAL_64)(children: Decimal64ColDivideDecimal64Scalar(col 2:decimal(7,2)/DECIMAL_64, decimal64Val 340, decimalVal 3.4) -> 15:decimal(12,6)/DECIMAL_64) -> 39:decimal(12,6)) -> 16:decimal(23,9), DecimalColModuloDecimalScalar(col 17:decimal(10,3), val 10)(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 17:decimal(10,3)) -> 18:decimal(5,3), CastDecimalToLong(col 19:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 19:decimal(10,3)) -> 20:int, CastDecimalToLong(col 21:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 21:decimal(7,2)) -> 22:smallint, CastDecimalToLong(col 23:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 23:decimal(7,2)) -> 24:tinyint, CastDecimalToLong(col 25:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 25:decimal(10,3)) -> 26:bigint, CastDecimalToBoolean(col 27:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 27:decimal(10,3)) -> 28:boolean, CastDecimalToDouble(col 29:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 29:decimal(7,2)) -> 30:double, CastDecimalToFloat(col 31:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 31:decimal(10,3)) -> 32:float, CastDecimalToString(col 33:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 33:decimal(7,2)) -> 34:string, CastDecimalToTimestamp(col 35:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 35:decimal(10,3)) -> 36:timestamp Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(11,3)), _col1 (type: decimal(11,3)), _col2 (type: decimal(21,11)), _col3 (type: decimal(23,9)), _col4 (type: decimal(5,3)), _col5 (type: int), _col6 (type: smallint), _col7 (type: tinyint), _col8 (type: bigint), _col9 (type: boolean), _col10 (type: double), _col11 (type: float), _col12 (type: string), _col13 (type: timestamp) @@ -253,7 +253,7 @@ STAGE PLANS: includeColumns: [0, 1, 2] dataColumns: cdouble:double, cdecimal1:decimal(10,3)/DECIMAL_64, cdecimal2:decimal(7,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(10,3), decimal(10,3), decimal(7,2), decimal(11,3), decimal(10,3), decimal(7,2), decimal(9,2), decimal(11,3), decimal(11,3)/DECIMAL_64, decimal(7,2), decimal(21,11), decimal(10,3), decimal(12,6)/DECIMAL_64, decimal(23,9), decimal(10,3), decimal(5,3), decimal(10,3), bigint, decimal(7,2), bigint, decimal(7,2), bigint, decimal(10,3), bigint, decimal(10,3), bigint, decimal(7,2), double, decimal(10,3), double, decimal(7,2), string, decimal(10,3), timestamp, decimal(11,3), decimal(12,6)] + scratchColumnTypeNames: [decimal(10,3), decimal(10,3), decimal(7,2), decimal(11,3), decimal(10,3), decimal(9,2)/DECIMAL_64, decimal(11,3), decimal(11,3)/DECIMAL_64, decimal(7,2), decimal(21,11), decimal(10,3), decimal(12,6)/DECIMAL_64, decimal(23,9), decimal(10,3), decimal(5,3), decimal(10,3), bigint, decimal(7,2), bigint, decimal(7,2), bigint, decimal(10,3), bigint, decimal(10,3), bigint, decimal(7,2), double, decimal(10,3), double, decimal(7,2), string, decimal(10,3), timestamp, decimal(9,2), decimal(11,3), decimal(12,6)] Reduce Vectorization: enabled: false enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true diff --git a/vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java b/vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java index 82fc1415de..9f1fff1dba 100644 --- a/vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java +++ b/vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java @@ -309,12 +309,15 @@ {"Decimal64ColumnArithmeticDecimal64Scalar", "Add", "+"}, {"Decimal64ColumnArithmeticDecimal64Scalar", "Subtract", "-"}, + {"Decimal64ColumnArithmeticDecimal64Scalar", "Multiply", "*"}, {"Decimal64ScalarArithmeticDecimal64Column", "Add", "+"}, {"Decimal64ScalarArithmeticDecimal64Column", "Subtract", "-"}, + {"Decimal64ScalarArithmeticDecimal64Column", "Multiply", "*"}, {"Decimal64ColumnArithmeticDecimal64Column", "Add", "+"}, {"Decimal64ColumnArithmeticDecimal64Column", "Subtract", "-"}, + {"Decimal64ColumnArithmeticDecimal64Column", "Multiply", "*"}, {"Decimal64ColumnDivideDecimal64Scalar", "Divide", "/"}, {"Decimal64ColumnDivideDecimal64Column", "Divide", "/"}, @@ -2535,22 +2538,32 @@ private void generateColumnArithmeticColumn(String [] tdesc) throws Exception { private void generateDecimal64ColumnArithmeticDecimal64Scalar(String[] tdesc) throws IOException { String operatorName = tdesc[1]; String className = "Decimal64Col" + operatorName + "Decimal64Scalar"; - generateDecimal64ColumnArithmetic(tdesc, className); + generateDecimal64ColumnArithmetic(tdesc, className, /* parentClassName */ null); + if (operatorName.equals("Multiply")) { + tdesc[0] = tdesc[0] + "Unscaled"; + String unscaledClassName = className + "Unscaled"; + generateDecimal64ColumnArithmetic(tdesc, unscaledClassName, className); + } } private void generateDecimal64ScalarArithmeticDecimal64Column(String[] tdesc) throws IOException { String operatorName = tdesc[1]; String className = "Decimal64Scalar" + operatorName + "Decimal64Column"; - generateDecimal64ColumnArithmetic(tdesc, className); + generateDecimal64ColumnArithmetic(tdesc, className, /* parentClassName */ null); + if (operatorName.equals("Multiply")) { + tdesc[0] = tdesc[0] + "Unscaled"; + String unscaledClassName = className + "Unscaled"; + generateDecimal64ColumnArithmetic(tdesc, unscaledClassName, className); + } } private void generateDecimal64ColumnArithmeticDecimal64Column(String[] tdesc) throws IOException { String operatorName = tdesc[1]; String className = "Decimal64Col" + operatorName + "Decimal64Column"; - generateDecimal64ColumnArithmetic(tdesc, className); + generateDecimal64ColumnArithmetic(tdesc, className, /* parentClassName */ null); } - private void generateDecimal64ColumnArithmetic(String[] tdesc, String className) + private void generateDecimal64ColumnArithmetic(String[] tdesc, String className, String parentClassName) throws IOException { String operatorSymbol = tdesc[2]; @@ -2560,6 +2573,9 @@ private void generateDecimal64ColumnArithmetic(String[] tdesc, String className) String templateString = readFile(templateFile); // Expand, and write result + if (parentClassName != null) { + templateString = templateString.replaceAll("", parentClassName); + } templateString = templateString.replaceAll("", className); templateString = templateString.replaceAll("", operatorSymbol); writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,