From 44ed57fbaa232b24b6f21dacc9ffe1edde184cb9 Mon Sep 17 00:00:00 2001 From: Owen O'Malley Date: Fri, 4 Aug 2017 15:34:43 -0700 Subject: [PATCH] HIVE-17235. Add Decimal64ColumnVector. --- .../hive/ql/exec/vector/Decimal64ColumnVector.java | 242 +++++++++++++++++++++ .../ql/exec/vector/TestDecimal64ColumnVector.java | 75 +++++++ 2 files changed, 317 insertions(+) create mode 100644 storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java create mode 100644 storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestDecimal64ColumnVector.java diff --git storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java new file mode 100644 index 0000000000..714328f3d5 --- /dev/null +++ storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java @@ -0,0 +1,242 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec.vector; + +import java.util.Arrays; + +/** + * This class represents a decimal column vector with precision <= 18. + * + * It uses a 64 bit integer to represent the integer representation of the + * decimal value. The + * + * The vector[] field is public by design for high-performance access in the inner + * loop of query execution. + */ +public class Decimal64ColumnVector extends ColumnVector { + public long[] vector; + public long precision; + public long scale; + + /** + * Use this constructor by default. All column vectors + * should normally be the default size. + */ + public Decimal64ColumnVector() { + this(VectorizedRowBatch.DEFAULT_SIZE, 38, 10); + } + + /** + * Don't use this except for testing purposes. + * + * @param len the number of rows + */ + public Decimal64ColumnVector(int len, int precision, int scale) { + super(len); + vector = new long[len]; + this.precision = precision; + this.scale = scale; + } + + // Copy the current object contents into the output. Only copy selected entries, + // as indicated by selectedInUse and the sel array. + public void copySelected( + boolean selectedInUse, int[] sel, int size, Decimal64ColumnVector output) { + + // Output has nulls if and only if input has nulls. + output.noNulls = noNulls; + output.isRepeating = false; + + // Handle repeating case + if (isRepeating) { + output.vector[0] = vector[0]; + output.isNull[0] = isNull[0]; + output.isRepeating = true; + return; + } + + // Handle normal case + + // Copy data values over + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.vector[i] = vector[i]; + } + } + else { + System.arraycopy(vector, 0, output.vector, 0, size); + } + + // Copy nulls over if needed + if (!noNulls) { + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.isNull[i] = isNull[i]; + } + } + else { + System.arraycopy(isNull, 0, output.isNull, 0, size); + } + } + } + + // Copy the current object contents into the output. Only copy selected entries, + // as indicated by selectedInUse and the sel array. + public void copySelected( + boolean selectedInUse, int[] sel, int size, DoubleColumnVector output) { + + // Output has nulls if and only if input has nulls. + output.noNulls = noNulls; + output.isRepeating = false; + + // Handle repeating case + if (isRepeating) { + output.vector[0] = vector[0]; // automatic conversion to double is done here + output.isNull[0] = isNull[0]; + output.isRepeating = true; + return; + } + + // Handle normal case + + // Copy data values over + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.vector[i] = vector[i]; + } + } + else { + for(int i = 0; i < size; ++i) { + output.vector[i] = vector[i]; + } + } + + // Copy nulls over if needed + if (!noNulls) { + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.isNull[i] = isNull[i]; + } + } + else { + System.arraycopy(isNull, 0, output.isNull, 0, size); + } + } + } + + // Fill the column vector with the provided value + public void fill(long value) { + noNulls = true; + isRepeating = true; + vector[0] = value; + } + + // Fill the column vector with nulls + public void fillWithNulls() { + noNulls = false; + isRepeating = true; + isNull[0] = true; + } + + // Simplify vector by brute-force flattening noNulls and isRepeating + // This can be used to reduce combinatorial explosion of code paths in VectorExpressions + // with many arguments. + public void flatten(boolean selectedInUse, int[] sel, int size) { + flattenPush(); + if (isRepeating) { + isRepeating = false; + long repeatVal = vector[0]; + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + vector[i] = repeatVal; + } + } else { + Arrays.fill(vector, 0, size, repeatVal); + } + flattenRepeatingNulls(selectedInUse, sel, size); + } + flattenNoNulls(selectedInUse, sel, size); + } + + @Override + public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) { + if (inputVector.isRepeating) { + inputElementNum = 0; + } + if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) { + isNull[outElementNum] = false; + vector[outElementNum] = + ((Decimal64ColumnVector) inputVector).vector[inputElementNum]; + } else { + isNull[outElementNum] = true; + noNulls = false; + } + } + + @Override + public void stringifyValue(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + String str = Long.toString(vector[row]); + int digits = str.length(); + if (digits > scale) { + buffer.append(str.substring(0, (int) (digits - scale))); + buffer.append('.'); + buffer.append(str.substring((int) (digits - scale))); + } else { + buffer.append("0."); + for(int d=digits; d < scale; ++d) { + buffer.append('0'); + } + buffer.append(str); + } + } else { + buffer.append("null"); + } + } + + @Override + public void ensureSize(int size, boolean preserveData) { + super.ensureSize(size, preserveData); + if (size > vector.length) { + long[] oldArray = vector; + vector = new long[size]; + if (preserveData) { + if (isRepeating) { + vector[0] = oldArray[0]; + } else { + System.arraycopy(oldArray, 0, vector, 0 , oldArray.length); + } + } + } + } + + @Override + public void shallowCopyTo(ColumnVector otherCv) { + Decimal64ColumnVector other = (Decimal64ColumnVector)otherCv; + super.shallowCopyTo(other); + other.vector = vector; + } +} diff --git storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestDecimal64ColumnVector.java storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestDecimal64ColumnVector.java new file mode 100644 index 0000000000..746b7620bb --- /dev/null +++ storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestDecimal64ColumnVector.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +public class TestDecimal64ColumnVector { + + @Test + public void testEnsureSize() { + Decimal64ColumnVector col = new Decimal64ColumnVector(); + for (int r = 0; r < 1000; ++r) { + col.vector[r] = r; + } + long[] vector = col.vector; + col.ensureSize(500, true); + assertEquals(vector, col.vector); + col.ensureSize(2048, true); + assertNotEquals(vector, col.vector); + for (int r = 0; r < 1000; ++r) { + assertEquals(r, col.vector[r]); + } + } + + @Test + public void testStringify() { + Decimal64ColumnVector col = new Decimal64ColumnVector(); + for (int r = 0; r < 1000; ++r) { + col.vector[r] = r; + } + col.scale = 2; + assertEquals("0.00", toString(col, 0)); + assertEquals("0.01", toString(col, 1)); + assertEquals("0.09", toString(col, 9)); + assertEquals("0.10", toString(col, 10)); + assertEquals("1.10", toString(col, 110)); + assertEquals("1.23", toString(col, 123)); + + col.noNulls = false; + col.isNull[9] = true; + col.scale = 1; + assertEquals("0.0", toString(col, 0)); + assertEquals("0.1", toString(col, 1)); + assertEquals("null", toString(col, 9)); + assertEquals("1.0", toString(col, 10)); + assertEquals("11.0", toString(col, 110)); + assertEquals("12.3", toString(col, 123)); + } + + static String toString(Decimal64ColumnVector cv, int row) { + StringBuilder buffer = new StringBuilder(); + cv.stringifyValue(buffer, row); + return buffer.toString(); + } +} -- 2.13.3