diff --git a/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java b/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java index bee0be1..5137784 100644 --- a/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java +++ b/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java @@ -107,6 +107,10 @@ {"ColumnDivideColumn", "Modulo", "double", "long", "%"}, {"ColumnDivideColumn", "Modulo", "double", "double", "%"}, + {"ColumnArithmeticScalarDecimal", "Add", "+"}, + {"ColumnArithmeticScalarDecimal", "Subtract", "-"}, + {"ColumnArithmeticScalarDecimal", "Multiply", "*"}, + {"ColumnCompareScalar", "Equal", "long", "double", "=="}, {"ColumnCompareScalar", "Equal", "double", "double", "=="}, {"ColumnCompareScalar", "NotEqual", "long", "double", "!="}, @@ -539,6 +543,8 @@ private void generate() throws Exception { for (String [] tdesc : templateExpansions) { if (tdesc[0].equals("ColumnArithmeticScalar") || tdesc[0].equals("ColumnDivideScalar")) { generateColumnArithmeticScalar(tdesc); + } else if (tdesc[0].equals("ColumnArithmeticScalarDecimal")) { + generateColumnArithmeticScalarDecimal(tdesc); } else if (tdesc[0].equals("ColumnCompareScalar")) { generateColumnCompareScalar(tdesc); } else if (tdesc[0].equals("ScalarCompareColumn")) { @@ -1116,6 +1122,20 @@ private void generateColumnArithmeticScalar(String[] tdesc) throws IOException { generateColumnBinaryOperatorScalar(tdesc, returnType, className); } + private void generateColumnArithmeticScalarDecimal(String[] tdesc) throws IOException { + String operatorName = tdesc[1]; + String className = "DecimalCol" + operatorName + "DecimalScalar"; + + //Read the template into a string; + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", operatorName.toLowerCase()); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + private void generateScalarArithmeticColumn(String[] tdesc) throws IOException { String operatorName = tdesc[1]; String operandType1 = tdesc[2]; diff --git a/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java b/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java index ec219fd..3939511 100644 --- a/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java +++ b/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java @@ -1604,4 +1604,13 @@ public String toString() { + signum + ", BigDecimal.toString=" + toBigDecimal().toString() + ", unscaledValue=[" + unscaledValue.toString() + "])"; } + + /** + * Vectorized execution uses the smallest possible positive non-zero + * value to prevent possible later zero-divide exceptions. Set the field + * to this value (1 in the internal unsigned 128 bit int). + */ + public void setNullDataValue() { + unscaledValue.update(1, 0, 0, 0); + } } diff --git a/ql/src/gen/vectorization/ExpressionTemplates/ColumnArithmeticScalarDecimal.txt b/ql/src/gen/vectorization/ExpressionTemplates/ColumnArithmeticScalarDecimal.txt new file mode 100644 index 0000000..c0bdd58 --- /dev/null +++ b/ql/src/gen/vectorization/ExpressionTemplates/ColumnArithmeticScalarDecimal.txt @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; +import org.apache.hadoop.hive.ql.exec.vector.expressions.DecimalUtil; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.common.type.Decimal128; + +/** + * Generated from template ColumnArithmeticScalarDecimal.txt, which covers binary arithmetic + * expressions between a column and a scalar. + */ +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int colNum; + private Decimal128 value; + private int outputColumn; + + public (int colNum, Decimal128 value, int outputColumn) { + this.colNum = colNum; + this.value = value; + this.outputColumn = outputColumn; + } + + public () { + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + DecimalColumnVector inputColVector = (DecimalColumnVector) batch.cols[colNum]; + DecimalColumnVector outputColVector = (DecimalColumnVector) batch.cols[outputColumn]; + int[] sel = batch.selected; + boolean[] inputIsNull = inputColVector.isNull; + boolean[] outputIsNull = outputColVector.isNull; + outputColVector.noNulls = inputColVector.noNulls; + outputColVector.isRepeating = inputColVector.isRepeating; + int n = batch.size; + Decimal128[] vector = inputColVector.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector.noNulls) { + + /* Initialize output vector NULL values to false. This is necessary + * since the decimal operation may produce a NULL result even for + * a non-null input vector value, and convert the output vector + * to have noNulls = false; + */ + NullUtil.initOutputNullsToFalse(outputColVector, inputColVector.isRepeating, + batch.selectedInUse, sel, n); + } + + if (inputColVector.isRepeating) { + if (!inputColVector.noNulls) { + outputIsNull[0] = inputIsNull[0]; + } + + // The following may override a "false" null setting if an error or overflow occurs. + DecimalUtil.Checked(0, vector[0], value, outputColVector); + } else if (inputColVector.noNulls) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + DecimalUtil.Checked(i, vector[i], value, outputColVector); + } + } else { + for(int i = 0; i != n; i++) { + DecimalUtil.Checked(i, vector[i], value, outputColVector); + } + } + } else /* there are nulls */ { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputIsNull[i] = inputIsNull[i]; + + // The following may override a "false" null setting if an error or overflow occurs. + DecimalUtil.Checked(i, vector[i], value, outputColVector); + } + } else { + System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); + for(int i = 0; i != n; i++) { + + // The following may override a "false" null setting if an error or overflow occurs. + DecimalUtil.Checked(i, vector[i], value, outputColVector); + } + } + } + + NullUtil.setNullDataEntriesDecimal(outputColVector, batch.selectedInUse, sel, n); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "decimal"; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + public Decimal128 getValue() { + return value; + } + + public void setValue(Decimal128 value) { + this.value = value; + } + + public void setOutputColumn(int outputColumn) { + this.outputColumn = outputColumn; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("decimal"), + VectorExpressionDescriptor.ArgumentType.getType("decimal")) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java new file mode 100644 index 0000000..23564bb --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; +import org.apache.hadoop.hive.common.type.Decimal128; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.io.Writable; + +public class DecimalColumnVector extends ColumnVector { + + /** + * A vector if Decimal128 objects. These are mutable and have fairly + * efficient operations on them. This will make it faster to load + * column vectors and perform decimal vector operations with decimal- + * specific VectorExpressions. + * + * For high performance and easy access to this low-level structure, + * the fields are public by design (as they are in other ColumnVector + * types). + */ + public Decimal128[] vector; + public short scale; + public short precision; + + public DecimalColumnVector(int precision, int scale) { + super(VectorizedRowBatch.DEFAULT_SIZE); + this.precision = (short) precision; + this.scale = (short) scale; + final int len = VectorizedRowBatch.DEFAULT_SIZE; + vector = new Decimal128[len]; + for (int i = 0; i < len; i++) { + vector[i] = new Decimal128(0, this.scale); + } + } + + @Override + public Writable getWritableObject(int index) { + // TODO Auto-generated method stub + return null; + } + + @Override + public void flatten(boolean selectedInUse, int[] sel, int size) { + // TODO Auto-generated method stub + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColAddDecimalColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColAddDecimalColumn.java new file mode 100644 index 0000000..3da5d4f --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColAddDecimalColumn.java @@ -0,0 +1,170 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.common.type.Decimal128; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; +import org.apache.hadoop.hive.ql.exec.vector.*; + +/** + * Generated from template ColumnArithmeticColumn.txt, which covers binary arithmetic + * expressions between columns. + */ +public class DecimalColAddDecimalColumn extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int colNum1; + private int colNum2; + private int outputColumn; + + public DecimalColAddDecimalColumn(int colNum1, int colNum2, int outputColumn) { + this.colNum1 = colNum1; + this.colNum2 = colNum2; + this.outputColumn = outputColumn; + } + + public DecimalColAddDecimalColumn() { + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + DecimalColumnVector inputColVector1 = (DecimalColumnVector) batch.cols[colNum1]; + DecimalColumnVector inputColVector2 = (DecimalColumnVector) batch.cols[colNum2]; + DecimalColumnVector outputColVector = (DecimalColumnVector) batch.cols[outputColumn]; + int[] sel = batch.selected; + int n = batch.size; + Decimal128[] vector1 = inputColVector1.vector; + Decimal128[] vector2 = inputColVector2.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + outputColVector.isRepeating = + inputColVector1.isRepeating && inputColVector2.isRepeating + || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0] + || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0]; + + // Handle nulls first + NullUtil.propagateNullsColCol( + inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); + + /* Disregard nulls for processing. In other words, + * the arithmetic operation is performed even if one or + * more inputs are null. This is to improve speed by avoiding + * conditional checks in the inner loop. + */ + if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + DecimalUtil.addChecked(0, vector1[0], vector2[0], outputColVector); + } else if (inputColVector1.isRepeating) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + DecimalUtil.addChecked(i, vector1[0], vector2[i], outputColVector); + } + } else { + for(int i = 0; i != n; i++) { + DecimalUtil.addChecked(i, vector1[0], vector2[i], outputColVector); + } + } + } else if (inputColVector2.isRepeating) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + DecimalUtil.addChecked(i, vector1[i], vector2[0], outputColVector); + } + } else { + for(int i = 0; i != n; i++) { + DecimalUtil.addChecked(i, vector1[i], vector2[0], outputColVector); + } + } + } else { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + DecimalUtil.addChecked(i, vector1[i], vector2[i], outputColVector); + } + } else { + for(int i = 0; i != n; i++) { + DecimalUtil.addChecked(i, vector1[i], vector2[i], outputColVector); + } + } + } + + /* For the case when the output can have null values, follow + * the convention that the data values must be set to a specific non-zero + * value. This is to prevent possible later zero-divide errors + * in complex arithmetic expressions like col2 / (col1 - 1) + * in the case when some col1 entries are null. + */ + NullUtil.setNullDataEntriesDecimal(outputColVector, batch.selectedInUse, sel, n); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "long"; + } + + public int getColNum1() { + return colNum1; + } + + public void setColNum1(int colNum1) { + this.colNum1 = colNum1; + } + + public int getColNum2() { + return colNum2; + } + + public void setColNum2(int colNum2) { + this.colNum2 = colNum2; + } + + public void setOutputColumn(int outputColumn) { + this.outputColumn = outputColumn; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("decimal"), + VectorExpressionDescriptor.ArgumentType.getType("decimal")) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalUtil.java new file mode 100644 index 0000000..1356ab5 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalUtil.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.common.type.Decimal128; +import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; + +/** + * Utility functions for vector operations on decimal values. + */ +public class DecimalUtil { + + // Addition with overflow check. Overflow produces NULL output. + public static void addChecked(int i, Decimal128 left, Decimal128 right, + DecimalColumnVector outputColVector) { + try { + Decimal128.add(left, right, outputColVector.vector[i], outputColVector.scale); + outputColVector.vector[i].checkPrecisionOverflow(outputColVector.precision); + } catch (ArithmeticException e) { // catch on overflow + outputColVector.noNulls = false; + outputColVector.isNull[i] = true; + } + } + + // Subtraction with overflow check. Overflow produces NULL output. + public static void subtractChecked(int i, Decimal128 left, Decimal128 right, + DecimalColumnVector outputColVector) { + try { + Decimal128.subtract(left, right, outputColVector.vector[i], outputColVector.scale); + outputColVector.vector[i].checkPrecisionOverflow(outputColVector.precision); + } catch (ArithmeticException e) { // catch on overflow + outputColVector.noNulls = false; + outputColVector.isNull[i] = true; + } + } + + // Multiplication with overflow check. Overflow produces NULL output. + public static void multiplyChecked(int i, Decimal128 left, Decimal128 right, + DecimalColumnVector outputColVector) { + try { + Decimal128.multiply(left, right, outputColVector.vector[i], outputColVector.scale); + outputColVector.vector[i].checkPrecisionOverflow(outputColVector.precision); + } catch (ArithmeticException e) { // catch on overflow + outputColVector.noNulls = false; + outputColVector.isNull[i] = true; + } + } + + // Division with overflow/zero-divide check. Error produces NULL output. + // Remainder argument is necessary to match up with the Decimal128.divide() interface. + // It will be discarded so just pass in a dummy argument. + public static void divideChecked(int i, Decimal128 left, Decimal128 right, + DecimalColumnVector outputColVector, Decimal128 remainder) { + try { + Decimal128.divide(left, right, outputColVector.vector[i], remainder, outputColVector.scale); + outputColVector.vector[i].checkPrecisionOverflow(outputColVector.precision); + } catch (ArithmeticException e) { // catch on error + outputColVector.noNulls = false; + outputColVector.isNull[i] = true; + } + } + + // Modulo operator with overflow/zero-divide check. + // Quotient argument is necessary to match up with Decimal128.divide() interface. + // It will be discarded so just pass in a dummy argument. + public static void moduloChecked(int i, Decimal128 left, Decimal128 right, + DecimalColumnVector outputColVector, Decimal128 quotient) { + try { + Decimal128.divide(left, right, quotient, outputColVector.vector[i], outputColVector.scale); + outputColVector.vector[i].checkPrecisionOverflow(outputColVector.precision); + } catch (ArithmeticException e) { // catch on error + outputColVector.noNulls = false; + outputColVector.isNull[i] = true; + } + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java index cd4cf92..982528b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java @@ -18,6 +18,10 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; +import java.util.Arrays; + +import org.apache.hadoop.hive.common.type.Decimal128; +import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; @@ -56,6 +60,7 @@ public static void setNullDataEntriesLong( public static void setNullOutputEntriesColScalar( ColumnVector v, boolean selectedInUse, int[] sel, int n) { if (v instanceof DoubleColumnVector) { + // No need to set null data entries because the input NaN values // will automatically propagate to the output. return; @@ -285,4 +290,48 @@ public static void propagateNullsColCol(ColumnVector inputColVector1, } } } + + // Follow the convention that null decimal values are internally set to the smallest + // positive value available. Prevents accidental zero-divide later in expression + // evaluation. + public static void setNullDataEntriesDecimal( + DecimalColumnVector v, boolean selectedInUse, int[] sel, + int n) { + if (v.noNulls) { + return; + } else if (v.isRepeating && v.isNull[0]) { + v.vector[0].setNullDataValue(); + } else if (selectedInUse) { + for (int j = 0; j != n; j++) { + int i = sel[j]; + if(v.isNull[i]) { + v.vector[i].setNullDataValue(); + } + } + } else { + for (int i = 0; i != n; i++) { + if(v.isNull[i]) { + v.vector[i].setNullDataValue(); + } + } + } + } + + // Initialize any entries that could be used in an output vector to have false for null value. + public static void initOutputNullsToFalse(ColumnVector v, boolean isRepeating, boolean selectedInUse, + int[] sel, int n) { + if (v.isRepeating) { + v.isNull[0] = false; + return; + } + + if (selectedInUse) { + for (int j = 0; j != n; j++) { + int i = sel[j]; + v.isNull[i] = false; + } + } else { + Arrays.fill(v.isNull, 0, n, false); + } + } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmeticExpressions.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmeticExpressions.java index 3ca727f..3a18c2e 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmeticExpressions.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmeticExpressions.java @@ -23,17 +23,22 @@ import static org.junit.Assert.assertTrue; import junit.framework.Assert; +import org.apache.hadoop.hive.common.type.Decimal128; +import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.TestVectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.DecimalColMultiplyDecimalScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.DecimalColSubtractDecimalScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColAddLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColAddLongScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.DecimalColAddDecimalScalar; import org.apache.hadoop.hive.ql.exec.vector.util.VectorizedRowGroupGenUtil; import org.junit.Test; /** - * Unit tests for vectori arithmetic expressions. + * Unit tests for vectorized arithmetic expressions. */ public class TestVectorArithmeticExpressions { @@ -284,4 +289,142 @@ public void testLongColDivideLongColumn() { assertFalse(out.noNulls); assertFalse(out.isRepeating); } + + @Test + public void testDecimalColAddDecimalColumn() { + VectorizedRowBatch b = getVectorizedRowBatch3DecimalCols(); + VectorExpression expr = new DecimalColAddDecimalColumn(0, 1, 2); + DecimalColumnVector r = (DecimalColumnVector) b.cols[2]; + + // test without nulls + expr.evaluate(b); + assertTrue(r.vector[0].equals(new Decimal128("2.20", (short) 2))); + assertTrue(r.vector[1].equals(new Decimal128("-2.30", (short) 2))); + assertTrue(r.vector[2].equals(new Decimal128("1.00", (short) 2))); + + // test nulls propagation + b = getVectorizedRowBatch3DecimalCols(); + DecimalColumnVector c0 = (DecimalColumnVector) b.cols[0]; + c0.noNulls = false; + c0.isNull[0] = true; + r = (DecimalColumnVector) b.cols[2]; + expr.evaluate(b); + assertTrue(!r.noNulls && r.isNull[0]); + + // Verify null output data entry is not 0, but rather the value specified by design, + // which is the minimum non-0 value, 0.01 in this case. + assertTrue(r.vector[0].equals(new Decimal128("0.01", (short) 2))); + + // test that overflow produces NULL + b = getVectorizedRowBatch3DecimalCols(); + c0 = (DecimalColumnVector) b.cols[0]; + c0.vector[0].update("9999999999999999.99", (short) 2); // set to max possible value + r = (DecimalColumnVector) b.cols[2]; + expr.evaluate(b); // will cause overflow for result at position 0, must yield NULL + assertTrue(!r.noNulls && r.isNull[0]); + + // verify proper null output data value + assertTrue(r.vector[0].equals(new Decimal128("0.01", (short) 2))); + } + + /* Test decimal column to decimal scalar addition. This is used to cover all the + * cases used in the source code template ColumnArithmeticScalarDecimal.txt. + */ + @Test + public void testDecimalColAddDecimalScalar() { + VectorizedRowBatch b = getVectorizedRowBatch3DecimalCols(); + Decimal128 d = new Decimal128(1); + VectorExpression expr = new DecimalColAddDecimalScalar(0, d, 2); + + // test without nulls + expr.evaluate(b); + DecimalColumnVector r = (DecimalColumnVector) b.cols[2]; + assertTrue(r.vector[0].equals(new Decimal128("2.20", (short) 2))); + assertTrue(r.vector[1].equals(new Decimal128("-2.30", (short) 2))); + assertTrue(r.vector[2].equals(new Decimal128("1.00", (short) 2))); + + // test null propagation + b = getVectorizedRowBatch3DecimalCols(); + DecimalColumnVector in = (DecimalColumnVector) b.cols[0]; + r = (DecimalColumnVector) b.cols[2]; + in.noNulls = false; + in.isNull[0] = true; + expr.evaluate(b); + assertTrue(!r.noNulls); + assertTrue(r.isNull[0]); + + // test repeating case, no nulls + b = getVectorizedRowBatch3DecimalCols(); + in = (DecimalColumnVector) b.cols[0]; + in.isRepeating = true; + expr.evaluate(b); + r = (DecimalColumnVector) b.cols[2]; + assertTrue(r.isRepeating); + assertTrue(r.vector[0].equals(new Decimal128("2.20", (short) 2))); + + // test repeating case for null value + b = getVectorizedRowBatch3DecimalCols(); + in = (DecimalColumnVector) b.cols[0]; + in.isRepeating = true; + in.isNull[0] = true; + in.noNulls = false; + expr.evaluate(b); + r = (DecimalColumnVector) b.cols[2]; + assertTrue(r.isRepeating); + assertTrue(!r.noNulls); + assertTrue(r.isNull[0]); + } + + /* Spot check correctness of decimal column subtract decimal scalar. The case for + * addition checks all the cases for the template, so don't do that redundantly here. + */ + @Test + public void testDecimalColSubtractDecimalScalar() { + VectorizedRowBatch b = getVectorizedRowBatch3DecimalCols(); + Decimal128 d = new Decimal128(1); + VectorExpression expr = new DecimalColSubtractDecimalScalar(0, d, 2); + + // test without nulls + expr.evaluate(b); + DecimalColumnVector r = (DecimalColumnVector) b.cols[2]; + assertTrue(r.vector[0].equals(new Decimal128("0.20", (short) 2))); + assertTrue(r.vector[1].equals(new Decimal128("-4.30", (short) 2))); + assertTrue(r.vector[2].equals(new Decimal128("-1.00", (short) 2))); + } + + /* Spot check correctness of decimal column multiply decimal scalar. The case for + * addition checks all the cases for the template, so don't do that redundantly here. + */ + @Test + public void testDecimalColMultiplyDecimalScalar() { + VectorizedRowBatch b = getVectorizedRowBatch3DecimalCols(); + Decimal128 d = new Decimal128(2); + VectorExpression expr = new DecimalColMultiplyDecimalScalar(0, d, 2); + + // test without nulls + expr.evaluate(b); + DecimalColumnVector r = (DecimalColumnVector) b.cols[2]; + assertTrue(r.vector[0].equals(new Decimal128("2.40", (short) 2))); + assertTrue(r.vector[1].equals(new Decimal128("-6.60", (short) 2))); + assertTrue(r.vector[2].equals(new Decimal128("0", (short) 2))); + } + + private VectorizedRowBatch getVectorizedRowBatch3DecimalCols() { + VectorizedRowBatch b = new VectorizedRowBatch(3); + DecimalColumnVector v0, v1; + b.cols[0] = v0 = new DecimalColumnVector(18, 2); + b.cols[1] = v1 = new DecimalColumnVector(18, 2); + b.cols[2] = new DecimalColumnVector(18, 2); + v0.vector[0].update("1.20", (short) 2); + v0.vector[1].update("-3.30", (short) 2); + v0.vector[2].update("0", (short) 2); + + v1.vector[0].update("1.00", (short) 2); + v1.vector[1].update("1.00", (short) 2); + v1.vector[2].update("1.00", (short) 2); + + b.size = 3; + + return b; + } }