diff --git a/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java b/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java index 0777c65..59279ec 100644 --- a/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java +++ b/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java @@ -119,6 +119,9 @@ {"ColumnArithmeticColumnDecimal", "Subtract"}, {"ColumnArithmeticColumnDecimal", "Multiply"}, + {"ColumnDivideScalarDecimal", "Divide"}, + {"ColumnDivideScalarDecimal", "Modulo"}, + {"ColumnCompareScalar", "Equal", "long", "double", "=="}, {"ColumnCompareScalar", "Equal", "double", "double", "=="}, {"ColumnCompareScalar", "NotEqual", "long", "double", "!="}, @@ -557,6 +560,8 @@ private void generate() throws Exception { generateScalarArithmeticColumnDecimal(tdesc); } else if (tdesc[0].equals("ColumnArithmeticColumnDecimal")) { generateColumnArithmeticColumnDecimal(tdesc); + } else if (tdesc[0].equals("ColumnDivideScalarDecimal")) { + generateColumnDivideScalarDecimal(tdesc); } else if (tdesc[0].equals("ColumnCompareScalar")) { generateColumnCompareScalar(tdesc); } else if (tdesc[0].equals("ScalarCompareColumn")) { @@ -1176,6 +1181,20 @@ private void generateColumnArithmeticColumnDecimal(String[] tdesc) throws IOExce className, templateString); } + private void generateColumnDivideScalarDecimal(String[] tdesc) throws IOException { + String operatorName = tdesc[1]; + String className = "DecimalCol" + getInitialCapWord(operatorName) + "DecimalScalar"; + + // Read the template into a string; + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", operatorName.toLowerCase()); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + private void generateScalarArithmeticColumn(String[] tdesc) throws IOException { String operatorName = tdesc[1]; String operandType1 = tdesc[2]; @@ -1234,6 +1253,14 @@ static String getCamelCaseType(String type) { } } + /** + * Return the argument with the first letter capitalized + */ + private static String getInitialCapWord(String word) { + String firstLetterAsCap = word.substring(0, 1).toUpperCase(); + return firstLetterAsCap + word.substring(1); + } + private String getArithmeticReturnType(String operandType1, String operandType2) { if (operandType1.equals("double") || diff --git a/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java b/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java index 3939511..f83c469 100644 --- a/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java +++ b/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java @@ -1613,4 +1613,21 @@ public String toString() { public void setNullDataValue() { unscaledValue.update(1, 0, 0, 0); } + + /** + * Zero the fractional part of value. + * + * Argument scratch is + * needed to hold unused remainder output, to avoid need to + * create a new object. + */ + public void removeFractionPart(UnsignedInt128 scratch) { + short placesToRemove = this.getScale(); + if (placesToRemove == 0) { + return; + } + UnsignedInt128 powerTenDivisor = SqlMathUtil.POWER_TENS_INT128[placesToRemove]; + this.getUnscaledValue().divideDestructive(powerTenDivisor, scratch); + this.getUnscaledValue().scaleUpTenDestructive(placesToRemove); + } } diff --git a/ql/src/gen/vectorization/ExpressionTemplates/ColumnArithmeticScalarDecimal.txt b/ql/src/gen/vectorization/ExpressionTemplates/ColumnArithmeticScalarDecimal.txt index c0bdd58..8615a4e 100644 --- a/ql/src/gen/vectorization/ExpressionTemplates/ColumnArithmeticScalarDecimal.txt +++ b/ql/src/gen/vectorization/ExpressionTemplates/ColumnArithmeticScalarDecimal.txt @@ -116,8 +116,6 @@ public class extends VectorExpression { } } } - - NullUtil.setNullDataEntriesDecimal(outputColVector, batch.selectedInUse, sel, n); } @Override diff --git a/ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideScalarDecimal.txt b/ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideScalarDecimal.txt new file mode 100644 index 0000000..5bbb0c3 --- /dev/null +++ b/ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideScalarDecimal.txt @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; +import org.apache.hadoop.hive.ql.exec.vector.expressions.DecimalUtil; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.common.type.Decimal128; + +/** + * Generated from template ColumnDivideScalarDecimal.txt, which covers binary arithmetic + * expressions between a column and a scalar. + */ +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int colNum; + private Decimal128 value; + private int outputColumn; + private transient Decimal128 zero; // to hold constant 0 for later use + private transient Decimal128 scratch; // scratch variable to hold unused function output + + public (int colNum, Decimal128 value, int outputColumn) { + this.colNum = colNum; + this.value = value; + this.outputColumn = outputColumn; + } + + public () { + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + DecimalColumnVector inputColVector = (DecimalColumnVector) batch.cols[colNum]; + DecimalColumnVector outputColVector = (DecimalColumnVector) batch.cols[outputColumn]; + int[] sel = batch.selected; + boolean[] inputIsNull = inputColVector.isNull; + boolean[] outputIsNull = outputColVector.isNull; + outputColVector.noNulls = inputColVector.noNulls; + outputColVector.isRepeating = inputColVector.isRepeating; + int n = batch.size; + Decimal128[] vector = inputColVector.vector; + Decimal128[] outputVector = outputColVector.vector; + + // Initialize local variable to use as 0 value on first use. + if (zero == null) { + this.zero = new Decimal128(0, inputColVector.scale); + this.scratch = new Decimal128(0, outputColVector.scale); + } + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector.noNulls) { + + /* Initialize output vector NULL values to false. This is necessary + * since the decimal operation may produce a NULL result even for + * a non-null input vector value, and convert the output vector + * to have noNulls = false; + */ + NullUtil.initOutputNullsToFalse(outputColVector, inputColVector.isRepeating, + batch.selectedInUse, sel, n); + } + + + if (value.compareTo(zero) == 0) { + + // Denominator is zero, convert the batch to nulls + outputColVector.noNulls = false; + outputColVector.isRepeating = true; + outputIsNull[0] = true; + } else if (inputColVector.isRepeating) { + DecimalUtil.Checked(0, vector[0], value, outputColVector, scratch); + + // Even if there are no nulls, we always copy over entry 0. Simplifies code. + outputIsNull[0] = inputIsNull[0]; + } else if (inputColVector.noNulls) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + DecimalUtil.Checked(i, vector[i], value, outputColVector, scratch); + } + } else { + for(int i = 0; i != n; i++) { + DecimalUtil.Checked(i, vector[i], value, outputColVector, scratch); + } + } + } else /* there are nulls */ { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + DecimalUtil.Checked(i, vector[i], value, outputColVector, scratch); + outputIsNull[i] = inputIsNull[i]; + } + } else { + for(int i = 0; i != n; i++) { + DecimalUtil.Checked(i, vector[i], value, outputColVector, scratch); + } + System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); + } + } + + NullUtil.setNullDataEntriesDecimal(outputColVector, batch.selectedInUse, sel, n); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "decimal"; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + public Decimal128 getValue() { + return value; + } + + public void setValue(Decimal128 value) { + this.value = value; + } + + public void setOutputColumn(int outputColumn) { + this.outputColumn = outputColumn; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("decimal"), + VectorExpressionDescriptor.ArgumentType.getType("decimal")) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalUtil.java index 1356ab5..5c694a5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalUtil.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; import org.apache.hadoop.hive.common.type.Decimal128; +import org.apache.hadoop.hive.common.type.SqlMathUtil; +import org.apache.hadoop.hive.common.type.UnsignedInt128; import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; /** @@ -77,13 +79,42 @@ public static void divideChecked(int i, Decimal128 left, Decimal128 right, } // Modulo operator with overflow/zero-divide check. - // Quotient argument is necessary to match up with Decimal128.divide() interface. - // It will be discarded so just pass in a dummy argument. + // The remainder arg is not really the modulo result. It is a scratch + // variable needed to recieve the remainder resujlt from Decimal128.divide(). public static void moduloChecked(int i, Decimal128 left, Decimal128 right, - DecimalColumnVector outputColVector, Decimal128 quotient) { + DecimalColumnVector outputColVector, Decimal128 remainder) { try { - Decimal128.divide(left, right, quotient, outputColVector.vector[i], outputColVector.scale); - outputColVector.vector[i].checkPrecisionOverflow(outputColVector.precision); + Decimal128.divide(left, right, outputColVector.vector[i], remainder, outputColVector.scale); + + /* + * The definition of modulo (x % p) is: + * x - IntegerPart(x / p, resultScale) * p + * + * left is x + * right is p + */ + + Decimal128 d = outputColVector.vector[i]; + + // d is currently x / p (the quotient) + + // take integer part of it + + // Use the Unsigned value in the remainder for scratch space since it is not + // needed otherwise. + UnsignedInt128 scratch = remainder.getUnscaledValue(); + d.removeFractionPart(scratch); + + // multiply by p + d.multiplyDestructive(right, outputColVector.scale); + + // negate it + d.negateDestructive(); + + // add x to it + d.addDestructive(left, outputColVector.scale); + + d.checkPrecisionOverflow(outputColVector.precision); } catch (ArithmeticException e) { // catch on error outputColVector.noNulls = false; outputColVector.isNull[i] = true; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmeticExpressions.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmeticExpressions.java index a2a3a00..e5ba4a1 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmeticExpressions.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmeticExpressions.java @@ -29,6 +29,8 @@ import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.TestVectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.DecimalColDivideDecimalScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.DecimalColModuloDecimalScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.DecimalColMultiplyDecimalScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.DecimalColSubtractDecimalScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.DecimalColumnMultiplyDecimalColumn; @@ -463,6 +465,83 @@ public void testDecimalColAddDecimalScalar() { assertTrue(r.isNull[0]); } + /* Test decimal column to decimal scalar division. This is used to cover all the + * cases used in the source code template ColumnDivideScalarDecimal.txt. + * The template is used for division and modulo. + */ + @Test + public void testDecimalColDivideDecimalScalar() { + VectorizedRowBatch b = getVectorizedRowBatch3DecimalCols(); + Decimal128 d = new Decimal128("2.00", (short) 2); + VectorExpression expr = new DecimalColDivideDecimalScalar(0, d, 2); + + + // test without nulls + expr.evaluate(b); + DecimalColumnVector r = (DecimalColumnVector) b.cols[2]; + assertTrue(r.vector[0].equals(new Decimal128("0.60", (short) 2))); + assertTrue(r.vector[1].equals(new Decimal128("-1.65", (short) 2))); + assertTrue(r.vector[2].equals(new Decimal128("0", (short) 2))); + + // test null propagation + b = getVectorizedRowBatch3DecimalCols(); + DecimalColumnVector in = (DecimalColumnVector) b.cols[0]; + r = (DecimalColumnVector) b.cols[2]; + in.noNulls = false; + in.isNull[0] = true; + expr.evaluate(b); + assertTrue(!r.noNulls); + assertTrue(r.isNull[0]); + + // test repeating case, no nulls + b = getVectorizedRowBatch3DecimalCols(); + in = (DecimalColumnVector) b.cols[0]; + in.isRepeating = true; + expr.evaluate(b); + r = (DecimalColumnVector) b.cols[2]; + assertTrue(r.isRepeating); + assertTrue(r.vector[0].equals(new Decimal128("0.60", (short) 2))); + + // test repeating case for null value + b = getVectorizedRowBatch3DecimalCols(); + in = (DecimalColumnVector) b.cols[0]; + in.isRepeating = true; + in.isNull[0] = true; + in.noNulls = false; + expr.evaluate(b); + r = (DecimalColumnVector) b.cols[2]; + assertTrue(r.isRepeating); + assertTrue(!r.noNulls); + assertTrue(r.isNull[0]); + + // test that zero-divide produces null for all output values + b = getVectorizedRowBatch3DecimalCols(); + in = (DecimalColumnVector) b.cols[0]; + expr = new DecimalColDivideDecimalScalar(0, new Decimal128("0", (short) 2), 2); + expr.evaluate(b); + r = (DecimalColumnVector) b.cols[2]; + assertFalse(r.noNulls); + assertTrue(r.isNull[0]); + assertTrue(r.isRepeating); + } + + /* + * Spot check Decimal Modulo + */ + @Test + public void testDecimalColModuloDecimalScalar() { + VectorizedRowBatch b = getVectorizedRowBatch3DecimalCols(); + Decimal128 d = new Decimal128("2.00", (short) 2); + VectorExpression expr = new DecimalColModuloDecimalScalar(0, d, 2); + + // test without nulls + expr.evaluate(b); + DecimalColumnVector r = (DecimalColumnVector) b.cols[2]; + assertTrue(r.vector[0].equals(new Decimal128("1.20", (short) 2))); + assertTrue(r.vector[1].equals(new Decimal128("-1.30", (short) 2))); + assertTrue(r.vector[2].equals(new Decimal128("0", (short) 2))); + } + /* Spot check correctness of decimal column subtract decimal scalar. The case for * addition checks all the cases for the template, so don't do that redundantly here. */