diff --git itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/VectorizedComparisonBench.java itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/VectorizedComparisonBench.java index 4c616f6..d54d1fa 100644 --- itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/VectorizedComparisonBench.java +++ itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/VectorizedComparisonBench.java @@ -14,24 +14,7 @@ package org.apache.hive.benchmark.vectorization; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColEqualLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColEqualLongScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColGreaterEqualLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColGreaterEqualLongScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColGreaterLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColGreaterLongScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColLessEqualLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColLessEqualLongScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColLessLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColLessLongScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColNotEqualLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColNotEqualLongScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongScalarEqualLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongScalarGreaterEqualLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongScalarGreaterLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongScalarLessEqualLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongScalarLessLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongScalarNotEqualLongColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*; import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.runner.Runner; diff --git ql/src/gen/vectorization/ExpressionTemplates/ColumnCompareColumn.txt ql/src/gen/vectorization/ExpressionTemplates/ColumnCompareColumn.txt index 56d96b8..84b2869 100644 --- ql/src/gen/vectorization/ExpressionTemplates/ColumnCompareColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/ColumnCompareColumn.txt @@ -33,8 +33,8 @@ public class extends VectorExpression { private static final long serialVersionUID = 1L; - private final int colNum1; - private final int colNum2; + protected final int colNum1; + protected final int colNum2; public (int colNum1, int colNum2, int outputColumnNum) { super(outputColumnNum); diff --git ql/src/gen/vectorization/ExpressionTemplates/ColumnCompareScalar.txt ql/src/gen/vectorization/ExpressionTemplates/ColumnCompareScalar.txt index 9556066..640feb3 100644 --- ql/src/gen/vectorization/ExpressionTemplates/ColumnCompareScalar.txt +++ ql/src/gen/vectorization/ExpressionTemplates/ColumnCompareScalar.txt @@ -36,8 +36,8 @@ public class extends VectorExpression { private static final long serialVersionUID = 1L; - private final int colNum; - private final value; + protected final int colNum; + protected final value; public (int colNum, value, int outputColumnNum) { super(outputColumnNum); diff --git ql/src/gen/vectorization/ExpressionTemplates/Decimal64ColumnCompareDecimal64Column.txt ql/src/gen/vectorization/ExpressionTemplates/Decimal64ColumnCompareDecimal64Column.txt new file mode 100644 index 0000000..d260716 --- /dev/null +++ ql/src/gen/vectorization/ExpressionTemplates/Decimal64ColumnCompareDecimal64Column.txt @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + +/** + * Generated from template Decimal64ColumnCompareDecimal64Column.txt, which covers + * decimal64 comparison expressions between two columns, however output is not produced in + * a separate column. The selected vector of the input {@link VectorizedRowBatch} is updated + * for in-place filtering. + */ +public class extends { + + private static final long serialVersionUID = 1L; + + public (int colNum1, int colNum2, int outputColumnNum) { + super(colNum1, colNum2, outputColumnNum); + } + + public () { + super(); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.DECIMAL_64, + VectorExpressionDescriptor.ArgumentType.DECIMAL_64) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } +} \ No newline at end of file diff --git ql/src/gen/vectorization/ExpressionTemplates/Decimal64ColumnCompareDecimal64Scalar.txt ql/src/gen/vectorization/ExpressionTemplates/Decimal64ColumnCompareDecimal64Scalar.txt new file mode 100644 index 0000000..802b9a6 --- /dev/null +++ ql/src/gen/vectorization/ExpressionTemplates/Decimal64ColumnCompareDecimal64Scalar.txt @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; + +/** + * Generated from template Decimal64ColumnCompareDecimal64Scalar.txt, which covers decimal64 + * comparison expressions between a column and a scalar, however output is not produced in a + * separate column. The selected vector of the input {@link VectorizedRowBatch} is updated for + * in-place filtering. + */ +public class extends { + + private static final long serialVersionUID = 1L; + + public (int colNum, long value, int outputColumnNum) { + super(colNum, value, outputColumnNum); + } + + public () { + super(); + } + + @Override + public String vectorExpressionParameters() { + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) inputTypeInfos[0]; + HiveDecimalWritable writable = new HiveDecimalWritable(); + writable.deserialize64(value, decimalTypeInfo.scale()); + return getColumnParamString(0, colNum) + ", decimal64Val " + value + + ", decimalVal " + writable.toString(); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.DECIMAL_64, + VectorExpressionDescriptor.ArgumentType.DECIMAL_64) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); + } +} \ No newline at end of file diff --git ql/src/gen/vectorization/ExpressionTemplates/Decimal64ScalarCompareDecimal64Column.txt ql/src/gen/vectorization/ExpressionTemplates/Decimal64ScalarCompareDecimal64Column.txt new file mode 100644 index 0000000..c8b10b6 --- /dev/null +++ ql/src/gen/vectorization/ExpressionTemplates/Decimal64ScalarCompareDecimal64Column.txt @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; + +/** + * Generated from template Decimal64ScalarCompareDecimal64Column.txt, which covers decimal64 + * comparison expressions between a scalar and a column, however output is not produced in a + * separate column. The selected vector of the input {@link VectorizedRowBatch} is updated for + * in-place filtering. + */ +public class extends { + + private static final long serialVersionUID = 1L; + + public (long value, int colNum, int outputColumnNum) { + super(value, colNum, outputColumnNum); + } + + public () { + super(); + } + + @Override + public String vectorExpressionParameters() { + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) inputTypeInfos[1]; + HiveDecimalWritable writable = new HiveDecimalWritable(); + writable.deserialize64(value, decimalTypeInfo.scale()); + return "decimal64Val " + value + ", decimalVal " + writable.toString() + + ", " + getColumnParamString(1, colNum); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.DECIMAL_64, + VectorExpressionDescriptor.ArgumentType.DECIMAL_64) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.SCALAR, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } +} \ No newline at end of file diff --git ql/src/gen/vectorization/ExpressionTemplates/DecimalColumnCompareDecimalColumn.txt ql/src/gen/vectorization/ExpressionTemplates/DecimalColumnCompareDecimalColumn.txt new file mode 100644 index 0000000..1a68de9 --- /dev/null +++ ql/src/gen/vectorization/ExpressionTemplates/DecimalColumnCompareDecimalColumn.txt @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; +import org.apache.hadoop.hive.ql.exec.vector.*; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.metadata.HiveException; + +/** + * Generated from template DecimalColumnCompareDecimalColumn.txt, which covers comparision + * expressions between Decimal columns. + */ +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private final int colNum1; + private final int colNum2; + + public (int colNum1, int colNum2, int outputColumnNum) { + super(outputColumnNum); + this.colNum1 = colNum1; + this.colNum2 = colNum2; + } + + public () { + super(); + + // Dummy final assignments. + colNum1 = -1; + colNum2 = -1; + } + + @Override + public void evaluate(VectorizedRowBatch batch) throws HiveException { + + // return immediately if batch is empty + final int n = batch.size; + if (n == 0) { + return; + } + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + DecimalColumnVector inputColVector1 = (DecimalColumnVector) batch.cols[colNum1]; + HiveDecimalWritable[] vector1 = inputColVector1.vector; + + DecimalColumnVector inputColVector2 = (DecimalColumnVector) batch.cols[colNum2]; + HiveDecimalWritable[] vector2 = inputColVector2.vector; + + LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; + int[] sel = batch.selected; + long[] outputVector = outputColVector.vector; + + /* + * Propagate null values for a two-input operator and set isRepeating and noNulls appropriately. + */ + NullUtil.propagateNullsColCol( + inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); + + /* Disregard nulls for processing. In other words, + * the arithmetic operation is performed even if one or + * more inputs are null. This is to improve speed by avoiding + * conditional checks in the inner loop. + */ + if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + outputVector[0] = vector1[0].compareTo(vector2[0]) 0 ? 1 : 0; + } else if (inputColVector1.isRepeating) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = vector1[0].compareTo(vector2[i]) 0 ? 1 : 0; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = vector1[0].compareTo(vector2[i]) 0 ? 1 : 0; + } + } + } else if (inputColVector2.isRepeating) { + HiveDecimalWritable value2 = vector2[0]; + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = vector1[i].compareTo(value2) 0 ? 1 : 0; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = vector1[i].compareTo(value2) 0 ? 1 : 0; + } + } + } else { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = vector1[i].compareTo(vector2[i]) 0 ? 1 : 0; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = vector1[i].compareTo(vector2[i]) 0 ? 1 : 0; + } + } + } + + /* For the case when the output can have null values, follow + * the convention that the data values must be 1 for long and + * NaN for double. This is to prevent possible later zero-divide errors + * in complex arithmetic expressions like col2 / (col1 - 1) + * in the case when some col1 entries are null. + */ + NullUtil.setNullDataEntriesLong(outputColVector, batch.selectedInUse, sel, n); + } + + @Override + public String vectorExpressionParameters() { + return getColumnParamString(0, colNum1) + ", " + getColumnParamString(1, colNum2); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.DECIMAL, + VectorExpressionDescriptor.ArgumentType.DECIMAL) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } +} diff --git ql/src/gen/vectorization/ExpressionTemplates/DecimalColumnCompareDecimalScalar.txt ql/src/gen/vectorization/ExpressionTemplates/DecimalColumnCompareDecimalScalar.txt new file mode 100644 index 0000000..8d09137 --- /dev/null +++ ql/src/gen/vectorization/ExpressionTemplates/DecimalColumnCompareDecimalScalar.txt @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import java.util.Arrays; + +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; +import org.apache.hadoop.hive.ql.exec.vector.*; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.metadata.HiveException; + +/** + * Generated from template DecimalColumnCompareDecimalScalar.txt, which covers binary comparison + * expressions between a column and a scalar. The boolean output is stored in a + * separate boolean column. + */ +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private final int colNum; + private final HiveDecimalWritable value; + + public (int colNum, HiveDecimal value, int outputColumnNum) { + super(outputColumnNum); + this.colNum = colNum; + this.value = new HiveDecimalWritable(value); + } + + public () { + super(); + + // Dummy final assignments. + colNum = -1; + value = null; + } + + @Override + public void evaluate(VectorizedRowBatch batch) throws HiveException { + + // return immediately if batch is empty + final int n = batch.size; + if (n == 0) { + return; + } + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + DecimalColumnVector inputColVector1 = (DecimalColumnVector) batch.cols[colNum]; + HiveDecimalWritable[] vector1 = inputColVector1.vector; + + LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; + + int[] sel = batch.selected; + boolean[] inputIsNull = inputColVector1.isNull; + boolean[] outputIsNull = outputColVector.isNull; + long[] outputVector = outputColVector.vector; + + // We do not need to do a column reset since we are carefully changing the output. + outputColVector.isRepeating = false; + + if (inputColVector1.isRepeating) { + if (inputColVector1.noNulls || !inputIsNull[0]) { + outputIsNull[0] = false; + outputVector[0] = vector1[0].compareTo(value) 0 ? 1 : 0; + } else { + outputIsNull[0] = true; + outputColVector.noNulls = false; + } + outputColVector.isRepeating = true; + NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n); + return; + } + + if (inputColVector1.noNulls) { + if (batch.selectedInUse) { + + // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop. + + if (!outputColVector.noNulls) { + for(int j = 0; j != n; j++) { + final int i = sel[j]; + outputIsNull[i] = false; + outputVector[i] = vector1[i].compareTo(value) 0 ? 1 : 0; + } + } else { + for(int j = 0; j != n; j++) { + final int i = sel[j]; + outputVector[i] = vector1[i].compareTo(value) 0 ? 1 : 0; + } + } + } else { + if (!outputColVector.noNulls) { + + // Assume it is almost always a performance win to fill all of isNull so we can + // safely reset noNulls. + Arrays.fill(outputIsNull, false); + outputColVector.noNulls = true; + } + for(int i = 0; i != n; i++) { + outputVector[i] = vector1[i].compareTo(value) 0 ? 1 : 0; + } + } + } else /* there are NULLs in the inputColVector */ { + + /* + * Do careful maintenance of the outputColVector.noNulls flag. + */ + + if (batch.selectedInUse) { + for(int j=0; j != n; j++) { + int i = sel[j]; + if (!inputIsNull[i]) { + outputIsNull[i] = false; + outputVector[i] = vector1[i].compareTo(value) 0 ? 1 : 0; + } else { + // Comparison with NULL is NULL. + outputIsNull[i] = true; + outputColVector.noNulls = false; + } + } + } else { + for(int i = 0; i != n; i++) { + if (!inputIsNull[i]) { + outputIsNull[i] = false; + outputVector[i] = vector1[i].compareTo(value) 0 ? 1 : 0; + } else { + // Comparison with NULL is NULL. + outputIsNull[i] = true; + outputColVector.noNulls = false; + } + } + } + } + } + + @Override + public String vectorExpressionParameters() { + return getColumnParamString(0, colNum) + ", val " + value.toString(); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.DECIMAL, + VectorExpressionDescriptor.ArgumentType.DECIMAL) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); + } +} diff --git ql/src/gen/vectorization/ExpressionTemplates/DecimalScalarCompareDecimalColumn.txt ql/src/gen/vectorization/ExpressionTemplates/DecimalScalarCompareDecimalColumn.txt new file mode 100644 index 0000000..1275260 --- /dev/null +++ ql/src/gen/vectorization/ExpressionTemplates/DecimalScalarCompareDecimalColumn.txt @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import java.util.Arrays; + +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.*; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.metadata.HiveException; + +/** + * Generated from template DecimalScalarCompareDecimalColumn.txt, which covers comparison + * expressions between a long/double scalar and a column. The boolean output is stored in a + * separate boolean column. + */ +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private final HiveDecimalWritable value; + private final int colNum; + + public (HiveDecimal value, int colNum, int outputColumnNum) { + super(outputColumnNum); + this.value = new HiveDecimalWritable(value); + this.colNum = colNum; + } + + public () { + super(); + + // Dummy final assignments. + value = null; + colNum = -1; + } + + @Override + public void evaluate(VectorizedRowBatch batch) throws HiveException { + + // return immediately if batch is empty + final int n = batch.size; + if (n == 0) { + return; + } + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + DecimalColumnVector inputColVector2 = (DecimalColumnVector) batch.cols[colNum]; + HiveDecimalWritable[] vector2 = inputColVector2.vector; + + LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; + + int[] sel = batch.selected; + boolean[] inputIsNull = inputColVector2.isNull; + boolean[] outputIsNull = outputColVector.isNull; + + long[] outputVector = outputColVector.vector; + + // We do not need to do a column reset since we are carefully changing the output. + outputColVector.isRepeating = false; + + if (inputColVector2.isRepeating) { + if (inputColVector2.noNulls || !inputIsNull[0]) { + outputIsNull[0] = false; + outputVector[0] = value.compareTo(vector2[0]) 0 ? 1 : 0; + } else { + outputIsNull[0] = true; + outputColVector.noNulls = false; + } + outputColVector.isRepeating = true; + NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n); + return; + } + + if (inputColVector2.noNulls) { + if (batch.selectedInUse) { + + // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop. + + if (!outputColVector.noNulls) { + for(int j = 0; j != n; j++) { + final int i = sel[j]; + outputIsNull[i] = false; + outputVector[i] = value.compareTo(vector2[i]) 0 ? 1 : 0; + } + } else { + for(int j = 0; j != n; j++) { + final int i = sel[j]; + outputVector[i] = value.compareTo(vector2[i]) 0 ? 1 : 0; + } + } + } else { + if (!outputColVector.noNulls) { + + // Assume it is almost always a performance win to fill all of isNull so we can + // safely reset noNulls. + Arrays.fill(outputIsNull, false); + outputColVector.noNulls = true; + } + for(int i = 0; i != n; i++) { + outputVector[i] = value.compareTo(vector2[i]) 0 ? 1 : 0; + } + } + } else /* there are NULLs in the inputColVector */ { + + /* + * Do careful maintenance of the outputColVector.noNulls flag. + */ + + if (batch.selectedInUse) { + for(int j=0; j != n; j++) { + int i = sel[j]; + if (!inputIsNull[i]) { + outputIsNull[i] = false; + outputVector[i] = value.compareTo(vector2[i]) 0 ? 1 : 0; + } else { + // Comparison with NULL is NULL. + outputIsNull[i] = true; + outputColVector.noNulls = false; + } + } + } else { + System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); + for(int i = 0; i != n; i++) { + if (!inputIsNull[i]) { + outputIsNull[i] = false; + outputVector[i] = value.compareTo(vector2[i]) 0 ? 1 : 0; + } else { + // Comparison with NULL is NULL. + outputIsNull[i] = true; + outputColVector.noNulls = false; + } + } + } + } + } + + @Override + public String vectorExpressionParameters() { + return "val " + value.toString() + ", " + getColumnParamString(1, colNum); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.DECIMAL, + VectorExpressionDescriptor.ArgumentType.DECIMAL) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.SCALAR, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } +} diff --git ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnScalar.txt ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnScalar.txt index d50a6f6..3eec95e 100644 --- ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnScalar.txt +++ ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnScalar.txt @@ -35,9 +35,9 @@ public class extends VectorExpression { private static final long serialVersionUID = 1L; - private final int arg1Column; - private final int arg2Column; - private final arg3Scalar; + protected final int arg1Column; + protected final int arg2Column; + protected final arg3Scalar; public (int arg1Column, int arg2Column, arg3Scalar, int outputColumnNum) { diff --git ql/src/gen/vectorization/ExpressionTemplates/IfExprObjectColumnColumn.txt ql/src/gen/vectorization/ExpressionTemplates/IfExprObjectColumnColumn.txt index e8ef279..a8ca31b 100644 --- ql/src/gen/vectorization/ExpressionTemplates/IfExprObjectColumnColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/IfExprObjectColumnColumn.txt @@ -23,6 +23,9 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.metadata.HiveException; +#IF DECIMAL +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +#ENDIF DECIMAL /** * Compute IF(expr1, expr2, expr3) for 3 input column expressions. @@ -64,8 +67,14 @@ public class extends VectorExpression { LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column]; arg2ColVector = () batch.cols[arg2Column]; boolean[] arg2IsNull = arg2ColVector.isNull; +#IF DECIMAL + HiveDecimalWritable[] vector2 = arg2ColVector.vector; +#ENDIF DECIMAL arg3ColVector = () batch.cols[arg3Column]; boolean[] arg3IsNull = arg3ColVector.isNull; +#IF DECIMAL + HiveDecimalWritable[] vector3 = arg3ColVector.vector; +#ENDIF DECIMAL outputColVector = () batch.cols[outputColumnNum]; int[] sel = batch.selected; boolean[] outputIsNull = outputColVector.isNull; @@ -107,7 +116,11 @@ public class extends VectorExpression { if (vector1[i] == 1) { if (!arg2IsNull[i]) { outputIsNull[i] = false; +#IF DECIMAL + outputColVector.set(i, vector2[i]); +#ELSE outputColVector.set(i, arg2ColVector.asScratch(i)); +#ENDIF DECIMAL } else { outputIsNull[i] = true; outputColVector.noNulls = false; @@ -115,7 +128,11 @@ public class extends VectorExpression { } else { if (!arg3IsNull[i]) { outputIsNull[i] = false; +#IF DECIMAL + outputColVector.set(i, vector3[i]); +#ELSE outputColVector.set(i, arg3ColVector.asScratch(i)); +#ENDIF DECIMAL } else { outputIsNull[i] = true; outputColVector.noNulls = false; @@ -127,7 +144,11 @@ public class extends VectorExpression { if (vector1[i] == 1) { if (!arg2IsNull[i]) { outputIsNull[i] = false; +#IF DECIMAL + outputColVector.set(i, vector2[i]); +#ELSE outputColVector.set(i, arg2ColVector.asScratch(i)); +#ENDIF DECIMAL } else { outputIsNull[i] = true; outputColVector.noNulls = false; @@ -135,7 +156,11 @@ public class extends VectorExpression { } else { if (!arg3IsNull[i]) { outputIsNull[i] = false; +#IF DECIMAL + outputColVector.set(i, vector3[i]); +#ELSE outputColVector.set(i, arg3ColVector.asScratch(i)); +#ENDIF DECIMAL } else { outputIsNull[i] = true; outputColVector.noNulls = false; @@ -150,7 +175,11 @@ public class extends VectorExpression { if (!arg1ColVector.isNull[i] && vector1[i] == 1) { if (!arg2IsNull[i]) { outputIsNull[i] = false; +#IF DECIMAL + outputColVector.set(i, vector2[i]); +#ELSE outputColVector.set(i, arg2ColVector.asScratch(i)); +#ENDIF DECIMAL } else { outputIsNull[i] = true; outputColVector.noNulls = false; @@ -158,7 +187,11 @@ public class extends VectorExpression { } else { if (!arg3IsNull[i]) { outputIsNull[i] = false; +#IF DECIMAL + outputColVector.set(i, vector3[i]); +#ELSE outputColVector.set(i, arg3ColVector.asScratch(i)); +#ENDIF DECIMAL } else { outputIsNull[i] = true; outputColVector.noNulls = false; @@ -170,7 +203,11 @@ public class extends VectorExpression { if (!arg1ColVector.isNull[i] && vector1[i] == 1) { if (!arg2IsNull[i]) { outputIsNull[i] = false; +#IF DECIMAL + outputColVector.set(i, vector2[i]); +#ELSE outputColVector.set(i, arg2ColVector.asScratch(i)); +#ENDIF DECIMAL } else { outputIsNull[i] = true; outputColVector.noNulls = false; @@ -178,7 +215,11 @@ public class extends VectorExpression { } else { if (!arg3IsNull[i]) { outputIsNull[i] = false; +#IF DECIMAL + outputColVector.set(i, vector3[i]); +#ELSE outputColVector.set(i, arg3ColVector.asScratch(i)); +#ENDIF DECIMAL } else { outputIsNull[i] = true; outputColVector.noNulls = false; diff --git ql/src/gen/vectorization/ExpressionTemplates/IfExprObjectColumnScalar.txt ql/src/gen/vectorization/ExpressionTemplates/IfExprObjectColumnScalar.txt index 56ae2ca..f0d2908 100644 --- ql/src/gen/vectorization/ExpressionTemplates/IfExprObjectColumnScalar.txt +++ ql/src/gen/vectorization/ExpressionTemplates/IfExprObjectColumnScalar.txt @@ -25,6 +25,9 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import ; import org.apache.hadoop.hive.ql.metadata.HiveException; +#IF DECIMAL +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +#ENDIF DECIMAL /** * Compute IF(expr1, expr2, expr3) for 3 input column expressions. @@ -67,6 +70,9 @@ public class extends VectorExpression { LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column]; arg2ColVector = () batch.cols[arg2Column]; boolean[] arg2IsNull = arg2ColVector.isNull; +#IF DECIMAL + HiveDecimalWritable[] vector2 = arg2ColVector.vector; +#ENDIF DECIMAL outputColVector = () batch.cols[outputColumnNum]; int[] sel = batch.selected; boolean[] outputIsNull = outputColVector.isNull; @@ -104,7 +110,11 @@ public class extends VectorExpression { if (vector1[i] == 1) { if (!arg2IsNull[i]) { outputIsNull[i] = false; +#IF DECIMAL + outputColVector.set(i, vector2[i]); +#ELSE outputColVector.set(i, arg2ColVector.asScratch(i)); +#ENDIF DECIMAL } else { outputIsNull[i] = true; outputColVector.noNulls = false; @@ -119,7 +129,11 @@ public class extends VectorExpression { if (vector1[i] == 1) { if (!arg2IsNull[i]) { outputIsNull[i] = false; +#IF DECIMAL + outputColVector.set(i, vector2[i]); +#ELSE outputColVector.set(i, arg2ColVector.asScratch(i)); +#ENDIF DECIMAL } else { outputIsNull[i] = true; outputColVector.noNulls = false; @@ -138,7 +152,11 @@ public class extends VectorExpression { if (!arg1ColVector.isNull[i] && vector1[i] == 1) { if (!arg2IsNull[i]) { outputIsNull[i] = false; +#IF DECIMAL + outputColVector.set(i, vector2[i]); +#ELSE outputColVector.set(i, arg2ColVector.asScratch(i)); +#ENDIF DECIMAL } else { outputIsNull[i] = true; outputColVector.noNulls = false; @@ -153,7 +171,11 @@ public class extends VectorExpression { if (!arg1ColVector.isNull[i] && vector1[i] == 1) { if (!arg2IsNull[i]) { outputIsNull[i] = false; +#IF DECIMAL + outputColVector.set(i, vector2[i]); +#ELSE outputColVector.set(i, arg2ColVector.asScratch(i)); +#ENDIF DECIMAL } else { outputIsNull[i] = true; outputColVector.noNulls = false; diff --git ql/src/gen/vectorization/ExpressionTemplates/IfExprObjectScalarColumn.txt ql/src/gen/vectorization/ExpressionTemplates/IfExprObjectScalarColumn.txt index 271b589..980f506 100644 --- ql/src/gen/vectorization/ExpressionTemplates/IfExprObjectScalarColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/IfExprObjectScalarColumn.txt @@ -25,6 +25,9 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import ; import org.apache.hadoop.hive.ql.metadata.HiveException; +#IF DECIMAL +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +#ENDIF DECIMAL /** * Compute IF(expr1, expr2, expr3) for 3 input column expressions. @@ -67,6 +70,9 @@ public class extends VectorExpression { LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column]; arg3ColVector = () batch.cols[arg3Column]; boolean[] arg3IsNull = arg3ColVector.isNull; +#IF DECIMAL + HiveDecimalWritable[] vector3 = arg3ColVector.vector; +#ENDIF DECIMAL outputColVector = () batch.cols[outputColumnNum]; int[] sel = batch.selected; boolean[] outputIsNull = outputColVector.isNull; @@ -109,7 +115,11 @@ public class extends VectorExpression { } else { if (!arg3IsNull[i]) { outputIsNull[i] = false; +#IF DECIMAL + outputColVector.set(i, vector3[i]); +#ELSE outputColVector.set(i, arg3ColVector.asScratch(i)); +#ENDIF DECIMAL } else { outputIsNull[i] = true; outputColVector.noNulls = false; @@ -124,7 +134,11 @@ public class extends VectorExpression { } else { if (!arg3IsNull[i]) { outputIsNull[i] = false; +#IF DECIMAL + outputColVector.set(i, vector3[i]); +#ELSE outputColVector.set(i, arg3ColVector.asScratch(i)); +#ENDIF DECIMAL } else { outputIsNull[i] = true; outputColVector.noNulls = false; @@ -143,7 +157,11 @@ public class extends VectorExpression { } else { if (!arg3IsNull[i]) { outputIsNull[i] = false; +#IF DECIMAL + outputColVector.set(i, vector3[i]); +#ELSE outputColVector.set(i, arg3ColVector.asScratch(i)); +#ENDIF DECIMAL } else { outputIsNull[i] = true; outputColVector.noNulls = false; @@ -158,7 +176,11 @@ public class extends VectorExpression { } else { if (!arg3IsNull[i]) { outputIsNull[i] = false; +#IF DECIMAL + outputColVector.set(i, vector3[i]); +#ELSE outputColVector.set(i, arg3ColVector.asScratch(i)); +#ENDIF DECIMAL } else { outputIsNull[i] = true; outputColVector.noNulls = false; diff --git ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarColumn.txt ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarColumn.txt index 3658129..a5a2957 100644 --- ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarColumn.txt @@ -35,9 +35,9 @@ public class extends VectorExpression { private static final long serialVersionUID = 1L; - private final int arg1Column; - private final arg2Scalar; - private final int arg3Column; + protected final int arg1Column; + protected final arg2Scalar; + protected final int arg3Column; public (int arg1Column, arg2Scalar, int arg3Column, int outputColumnNum) { diff --git ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarScalar.txt ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarScalar.txt index d11459a..20ce883 100644 --- ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarScalar.txt +++ ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarScalar.txt @@ -36,9 +36,9 @@ public class extends VectorExpression { private static final long serialVersionUID = 1L; - private final int arg1Column; - private final arg2Scalar; - private final arg3Scalar; + protected final int arg1Column; + protected final arg2Scalar; + protected final arg3Scalar; public (int arg1Column, arg2Scalar, arg3Scalar, int outputColumnNum) { diff --git ql/src/gen/vectorization/ExpressionTemplates/ScalarCompareColumn.txt ql/src/gen/vectorization/ExpressionTemplates/ScalarCompareColumn.txt index 753f061..60dc725 100644 --- ql/src/gen/vectorization/ExpressionTemplates/ScalarCompareColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/ScalarCompareColumn.txt @@ -37,8 +37,8 @@ public class extends VectorExpression { private static final long serialVersionUID = 1L; - private final value; - private final int colNum; + protected final value; + protected final int colNum; public ( value, int colNum, int outputColumnNum) { super(outputColumnNum); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java index 2f8a419..fb40f5e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java @@ -89,6 +89,8 @@ INT_INTERVAL_YEAR_MONTH (INT_FAMILY.value | INTERVAL_YEAR_MONTH.value), INT_DATE_INTERVAL_YEAR_MONTH (INT_FAMILY.value | DATE.value | INTERVAL_YEAR_MONTH.value), STRING_DATETIME_FAMILY (STRING_FAMILY.value | DATETIME_FAMILY.value), + STRING_FAMILY_BINARY (STRING_FAMILY.value | BINARY.value), + STRING_BINARY (STRING.value | BINARY.value), ALL_FAMILY (0xFFFFFFL); private final long value; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index 5631347..72c3052 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -1540,88 +1540,88 @@ private boolean checkTypeInfoForDecimal64(TypeInfo typeInfo) { return false; } - public boolean haveCandidateForDecimal64VectorExpression(int numChildren, - List childExpr, TypeInfo returnType) throws HiveException { - - // For now, just 2 Decimal64 inputs and a Decimal64 or boolean output. - return (numChildren == 2 && - checkExprNodeDescForDecimal64(childExpr.get(0)) && - checkExprNodeDescForDecimal64(childExpr.get(1)) && - (checkTypeInfoForDecimal64(returnType) || - returnType.equals(TypeInfoFactory.booleanTypeInfo))); - } - private VectorExpression getDecimal64VectorExpressionForUdf(GenericUDF genericUdf, - Class udfClass, List childExpr, int numChildren, - VectorExpressionDescriptor.Mode mode, TypeInfo returnType) throws HiveException { - - ExprNodeDesc child1 = childExpr.get(0); - ExprNodeDesc child2 = childExpr.get(1); - - DecimalTypeInfo decimalTypeInfo1 = (DecimalTypeInfo) child1.getTypeInfo(); - DecimalTypeInfo decimalTypeInfo2 = (DecimalTypeInfo) child2.getTypeInfo(); - - DataTypePhysicalVariation dataTypePhysicalVariation1 = DataTypePhysicalVariation.DECIMAL_64; - DataTypePhysicalVariation dataTypePhysicalVariation2 = DataTypePhysicalVariation.DECIMAL_64; - - final int scale1 = decimalTypeInfo1.scale(); - final int scale2 = decimalTypeInfo2.scale(); + Class udfClass, List childExprs, int numChildren, + VectorExpressionDescriptor.Mode mode, TypeInfo returnTypeInfo) throws HiveException { VectorExpressionDescriptor.Builder builder = new VectorExpressionDescriptor.Builder(); builder.setNumArguments(numChildren); builder.setMode(mode); - boolean isColumnScaleEstablished = false; - int columnScale = 0; - boolean hasScalar = false; - builder.setArgumentType(0, ArgumentType.DECIMAL_64); - if (child1 instanceof ExprNodeGenericFuncDesc || - child1 instanceof ExprNodeColumnDesc) { - builder.setInputExpressionType(0, InputExpressionType.COLUMN); - isColumnScaleEstablished = true; - columnScale = scale1; - } else if (child1 instanceof ExprNodeConstantDesc) { - if (isNullConst(child1)) { - - // Cannot handle NULL scalar parameter. + // DECIMAL_64 decimals must have same scale. + boolean anyDecimal64Expr = false; + boolean isDecimal64ScaleEstablished = false; + int decimal64ColumnScale = 0; + + for (int i = 0; i < numChildren; i++) { + ExprNodeDesc childExpr = childExprs.get(i); + + /* + * For columns, we check decimal columns for DECIMAL_64 DataTypePhysicalVariation. + * For UDFs, we check for @VectorizedExpressionsSupportDecimal64 annotation, etc. + */ + final boolean isExprDecimal64 = checkExprNodeDescForDecimal64(childExpr); + if (isExprDecimal64) { + anyDecimal64Expr = true; + } + + TypeInfo typeInfo = childExpr.getTypeInfo(); + if (childExpr instanceof ExprNodeGenericFuncDesc || + childExpr instanceof ExprNodeColumnDesc) { + if (isExprDecimal64) { + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo; + if (!isDecimal64ScaleEstablished) { + decimal64ColumnScale = decimalTypeInfo.getScale(); + isDecimal64ScaleEstablished = true; + } else if (decimalTypeInfo.getScale() != decimal64ColumnScale) { + return null; + } + } + builder.setInputExpressionType(i, InputExpressionType.COLUMN); + } else if (childExpr instanceof ExprNodeConstantDesc) { + if (isNullConst(childExpr)) { + // Cannot handle NULL scalar parameter. + return null; + } + builder.setInputExpressionType(i, InputExpressionType.SCALAR); + } else { return null; } - hasScalar = true; - builder.setInputExpressionType(0, InputExpressionType.SCALAR); - } else { - // Currently, only functions, columns, and scalars supported. - return null; + if (isExprDecimal64) { + builder.setArgumentType(i, ArgumentType.DECIMAL_64); + } else { + String undecoratedTypeName = getUndecoratedName(childExpr.getTypeString()); + if (undecoratedTypeName == null) { + return null; + } + builder.setArgumentType(i, undecoratedTypeName); + } } - builder.setArgumentType(1, ArgumentType.DECIMAL_64); - if (child2 instanceof ExprNodeGenericFuncDesc || - child2 instanceof ExprNodeColumnDesc) { - builder.setInputExpressionType(1, InputExpressionType.COLUMN); - if (!isColumnScaleEstablished) { - isColumnScaleEstablished = true; - columnScale = scale2; - } else if (columnScale != scale2) { - - // We only support Decimal64 on 2 columns when the have the same scale. - return null; - } - } else if (child2 instanceof ExprNodeConstantDesc) { - // Cannot have SCALAR, SCALAR. - if (!isColumnScaleEstablished) { - return null; - } - if (isNullConst(child2)) { + if (!anyDecimal64Expr) { + return null; + } - // Cannot handle NULL scalar parameter. + final boolean isReturnDecimal64 = checkTypeInfoForDecimal64(returnTypeInfo); + final DataTypePhysicalVariation returnDataTypePhysicalVariation; + if (isReturnDecimal64) { + DecimalTypeInfo returnDecimalTypeInfo = (DecimalTypeInfo) returnTypeInfo; + if (!isDecimal64ScaleEstablished) { + decimal64ColumnScale = returnDecimalTypeInfo.getScale(); + isDecimal64ScaleEstablished = true; + } else if (returnDecimalTypeInfo.getScale() != decimal64ColumnScale) { return null; } - hasScalar = true; - builder.setInputExpressionType(1, InputExpressionType.SCALAR); - } else { + returnDataTypePhysicalVariation = DataTypePhysicalVariation.DECIMAL_64; + } else if (returnTypeInfo instanceof DecimalTypeInfo){ - // Currently, only functions, columns, and scalars supported. + // Currently, we don't have any vectorized expressions that take DECIMAL_64 inputs + // and produce a regular decimal. Or, currently, a way to express that in the + // descriptor. return null; + } else { + returnDataTypePhysicalVariation = DataTypePhysicalVariation.NONE; } VectorExpressionDescriptor.Descriptor descriptor = builder.build(); @@ -1639,19 +1639,26 @@ private VectorExpression getDecimal64VectorExpressionForUdf(GenericUDF genericUd List children = new ArrayList(); Object[] arguments = new Object[numChildren]; + TypeInfo[] typeInfos = new TypeInfo[numChildren]; + DataTypePhysicalVariation[] dataTypePhysicalVariations = new DataTypePhysicalVariation[numChildren]; for (int i = 0; i < numChildren; i++) { - ExprNodeDesc child = childExpr.get(i); - if (child instanceof ExprNodeGenericFuncDesc) { - VectorExpression vChild = getVectorExpression(child, childrenMode); + ExprNodeDesc childExpr = childExprs.get(i); + TypeInfo typeInfo = childExpr.getTypeInfo(); + typeInfos[i] = typeInfo; + dataTypePhysicalVariations[i] = + (checkTypeInfoForDecimal64(typeInfo) ? + DataTypePhysicalVariation.DECIMAL_64 : DataTypePhysicalVariation.NONE); + if (childExpr instanceof ExprNodeGenericFuncDesc) { + VectorExpression vChild = getVectorExpression(childExpr, childrenMode); children.add(vChild); arguments[i] = vChild.getOutputColumnNum(); - } else if (child instanceof ExprNodeColumnDesc) { - int colIndex = getInputColumnIndex((ExprNodeColumnDesc) child); + } else if (childExpr instanceof ExprNodeColumnDesc) { + int colIndex = getInputColumnIndex((ExprNodeColumnDesc) childExpr); if (childrenMode == VectorExpressionDescriptor.Mode.FILTER) { VectorExpression filterExpr = - getFilterOnBooleanColumnExpression((ExprNodeColumnDesc) child, colIndex); + getFilterOnBooleanColumnExpression((ExprNodeColumnDesc) childExpr, colIndex); if (filterExpr == null) { return null; } @@ -1660,16 +1667,27 @@ private VectorExpression getDecimal64VectorExpressionForUdf(GenericUDF genericUd } arguments[i] = colIndex; } else { - Preconditions.checkState(child instanceof ExprNodeConstantDesc); - ExprNodeConstantDesc constDesc = (ExprNodeConstantDesc) child; - HiveDecimal hiveDecimal = (HiveDecimal) constDesc.getValue(); - if (hiveDecimal.scale() > columnScale) { + Preconditions.checkState(childExpr instanceof ExprNodeConstantDesc); + ExprNodeConstantDesc constDesc = (ExprNodeConstantDesc) childExpr; + if (typeInfo instanceof DecimalTypeInfo) { + if (!isDecimal64ScaleEstablished) { + return null; + } + HiveDecimal hiveDecimal = (HiveDecimal) constDesc.getValue(); + if (hiveDecimal.scale() > decimal64ColumnScale) { - // For now, bail out on decimal constants with larger scale than column scale. - return null; + // For now, bail out on decimal constants with larger scale than column scale. + return null; + } + final long decimal64Scalar = + new HiveDecimalWritable(hiveDecimal).serialize64(decimal64ColumnScale); + arguments[i] = decimal64Scalar; + } else { + Object scalarValue = getVectorTypeScalarValue(constDesc); + arguments[i] = + (scalarValue == null) ? + getConstantVectorExpression(null, typeInfo, childrenMode) : scalarValue; } - final long decimal64Scalar = new HiveDecimalWritable(hiveDecimal).serialize64(columnScale); - arguments[i] = decimal64Scalar; } } @@ -1679,13 +1697,13 @@ private VectorExpression getDecimal64VectorExpressionForUdf(GenericUDF genericUd * The instantiateExpression method sets the output column and type information. */ VectorExpression vectorExpression = - instantiateExpression(vectorClass, returnType, DataTypePhysicalVariation.DECIMAL_64, arguments); + instantiateExpression(vectorClass, returnTypeInfo, returnDataTypePhysicalVariation, arguments); if (vectorExpression == null) { - handleCouldNotInstantiateVectorExpression(vectorClass, returnType, DataTypePhysicalVariation.DECIMAL_64, arguments); + handleCouldNotInstantiateVectorExpression(vectorClass, returnTypeInfo, returnDataTypePhysicalVariation, arguments); } - vectorExpression.setInputTypeInfos(decimalTypeInfo1, decimalTypeInfo2); - vectorExpression.setInputDataTypePhysicalVariations(dataTypePhysicalVariation1, dataTypePhysicalVariation2); + vectorExpression.setInputTypeInfos(typeInfos); + vectorExpression.setInputDataTypePhysicalVariations(dataTypePhysicalVariations); if ((vectorExpression != null) && !children.isEmpty()) { vectorExpression.setChildExpressions(children.toArray(new VectorExpression[0])); @@ -1735,16 +1753,15 @@ private VectorExpression getVectorExpressionForUdf(GenericUDF genericUdf, return null; } - // Should we intercept here for a possible Decimal64 vector expression class? - if (haveCandidateForDecimal64VectorExpression(numChildren, childExpr, returnType)) { - VectorExpression result = getDecimal64VectorExpressionForUdf(genericUdf, udfClass, - childExpr, numChildren, mode, returnType); - if (result != null) { - return result; - } - // Otherwise, fall through and proceed with non-Decimal64 vector expression classes... + // Intercept here for a possible Decimal64 vector expression class. + VectorExpression result = getDecimal64VectorExpressionForUdf(genericUdf, udfClass, + childExpr, numChildren, mode, returnType); + if (result != null) { + return result; } + // Otherwise, fall through and proceed with non-Decimal64 vector expression classes... + VectorExpressionDescriptor.Builder builder = new VectorExpressionDescriptor.Builder(); builder.setNumArguments(numChildren); builder.setMode(mode); @@ -2067,6 +2084,8 @@ private VectorExpression getGenericUdfVectorExpression(GenericUDF udf, ve = getCastToChar(childExpr, returnType); } else if (udf instanceof GenericUDFToVarchar) { ve = getCastToVarChar(childExpr, returnType); + } else if (udf instanceof GenericUDFToBinary) { + ve = getCastToBinary(childExpr, returnType); } else if (udf instanceof GenericUDFTimestamp) { ve = getCastToTimestamp((GenericUDFTimestamp)udf, childExpr, mode, returnType); } @@ -2870,6 +2889,25 @@ private VectorExpression getCastToVarChar(List childExpr, TypeInfo return null; } + private VectorExpression getCastToBinary(List childExpr, TypeInfo returnType) + throws HiveException { + ExprNodeDesc child = childExpr.get(0); + String inputType = childExpr.get(0).getTypeString(); + if (child instanceof ExprNodeConstantDesc) { + // Don't do constant folding here. Wait until the optimizer is changed to do it. + // Family of related JIRAs: HIVE-7421, HIVE-7422, and HIVE-7424. + return null; + } + if (inputType.equalsIgnoreCase("string") || varcharTypePattern.matcher(inputType).matches()) { + + // STRING and VARCHAR types require no conversion, so use a no-op. + return getIdentityExpression(childExpr); + } else if (charTypePattern.matcher(inputType).matches()) { + return createVectorExpression(CastCharToBinary.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType); + } + return null; + } + private VectorExpression getCastToDoubleExpression(Class udf, List childExpr, TypeInfo returnType) throws HiveException { ExprNodeDesc child = childExpr.get(0); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastCharToBinary.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastCharToBinary.java new file mode 100644 index 0000000..f8f60a8 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastCharToBinary.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; + +public class CastCharToBinary extends StringUnaryUDFDirect { + + private static final long serialVersionUID = 1L; + private int maxLength; + + public CastCharToBinary(int inputColumn, int outputColumnNum) { + super(inputColumn, outputColumnNum); + } + + public CastCharToBinary() { + super(); + } + + @Override + public void transientInit() throws HiveException { + super.transientInit(); + + maxLength = ((CharTypeInfo) inputTypeInfos[0]).getLength(); + } + + /** + * Do pad out the CHAR type into the BINARY result, taking into account Unicode... + */ + protected void func(BytesColumnVector outV, byte[][] vector, int[] start, int[] length, int i) { + StringExpr.padRight(outV, i, vector[i], start[i], length[i], maxLength); + } + + public String vectorExpressionParameters() { + return getColumnParamString(0, inputColumn) + ", maxLength " + maxLength; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToTimestamp.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToTimestamp.java new file mode 100644 index 0000000..b48b013 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToTimestamp.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import java.util.Arrays; +import java.sql.Timestamp; + +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + + +/** + * Casts a string vector to a Timestamp vector. + */ +public class CastStringToTimestamp extends VectorExpression { + private static final long serialVersionUID = 1L; + + private final int inputColumn; + + public CastStringToTimestamp() { + super(); + + // Dummy final assignments. + inputColumn = -1; + } + + public CastStringToTimestamp(int inputColumn, int outputColumnNum) { + super(outputColumnNum); + this.inputColumn = inputColumn; + } + + @Override + public void evaluate(VectorizedRowBatch batch) throws HiveException { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + BytesColumnVector inputColVector = (BytesColumnVector) batch.cols[inputColumn]; + int[] sel = batch.selected; + int n = batch.size; + TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumnNum]; + + boolean[] inputIsNull = inputColVector.isNull; + boolean[] outputIsNull = outputColVector.isNull; + + if (n == 0) { + + // Nothing to do + return; + } + + // We do not need to do a column reset since we are carefully changing the output. + outputColVector.isRepeating = false; + + if (inputColVector.isRepeating) { + if (inputColVector.noNulls || !inputIsNull[0]) { + // Set isNull before call in case it changes it mind. + outputIsNull[0] = false; + evaluate(outputColVector, inputColVector, 0); + } else { + outputIsNull[0] = true; + outputColVector.noNulls = false; + } + outputColVector.isRepeating = true; + return; + } + + if (inputColVector.noNulls) { + if (batch.selectedInUse) { + + // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop. + + if (!outputColVector.noNulls) { + for(int j = 0; j != n; j++) { + final int i = sel[j]; + // Set isNull before call in case it changes it mind. + outputIsNull[i] = false; + evaluate(outputColVector, inputColVector, i); + } + } else { + for(int j = 0; j != n; j++) { + final int i = sel[j]; + evaluate(outputColVector, inputColVector, i); + } + } + } else { + if (!outputColVector.noNulls) { + + // Assume it is almost always a performance win to fill all of isNull so we can + // safely reset noNulls. + Arrays.fill(outputIsNull, false); + outputColVector.noNulls = true; + } + for(int i = 0; i != n; i++) { + evaluate(outputColVector, inputColVector, i); + } + } + } else /* there are NULLs in the inputColVector */ { + + // Carefully handle NULLs... + + outputColVector.noNulls = false; + + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + // Set isNull before call in case it changes it mind. + outputColVector.isNull[i] = inputColVector.isNull[i]; + if (!inputColVector.isNull[i]) { + evaluate(outputColVector, inputColVector, i); + } + } + } else { + // Set isNull before calls in case they change their mind. + System.arraycopy(inputColVector.isNull, 0, outputColVector.isNull, 0, n); + for(int i = 0; i != n; i++) { + if (!inputColVector.isNull[i]) { + evaluate(outputColVector, inputColVector, i); + } + } + } + } + } + + private void evaluate(TimestampColumnVector outputColVector, BytesColumnVector inputColVector, int i) { + try { + org.apache.hadoop.hive.common.type.Timestamp timestamp = + PrimitiveObjectInspectorUtils.getTimestampFromString( + new String( + inputColVector.vector[i], inputColVector.start[i], inputColVector.length[i], + "UTF-8")); + outputColVector.set(i, timestamp.toSqlTimestamp()); + } catch (Exception e) { + outputColVector.setNullValue(i); + outputColVector.isNull[i] = true; + outputColVector.noNulls = false; + } + } + + @Override + public String vectorExpressionParameters() { + return getColumnParamString(0, inputColumn); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + VectorExpressionDescriptor.Builder b = new VectorExpressionDescriptor.Builder(); + b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(1) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN); + return b.build(); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprCharScalarStringScalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprCharScalarStringScalar.java index 0b0c532..256cb8d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprCharScalarStringScalar.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprCharScalarStringScalar.java @@ -48,7 +48,7 @@ public IfExprCharScalarStringScalar() { .setArgumentTypes( VectorExpressionDescriptor.ArgumentType.INT_FAMILY, VectorExpressionDescriptor.ArgumentType.CHAR, - VectorExpressionDescriptor.ArgumentType.STRING) + VectorExpressionDescriptor.ArgumentType.CHAR) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.SCALAR, diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprDecimal64ColumnDecimal64Column.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprDecimal64ColumnDecimal64Column.java new file mode 100644 index 0000000..0cf2ffe --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprDecimal64ColumnDecimal64Column.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + +/** + * Compute IF(expr1, expr2, expr3) for 3 input column expressions. + * The first is always a boolean (LongColumnVector). + * The second and third are long columns or long expression results. + */ +public class IfExprDecimal64ColumnDecimal64Column extends IfExprLongColumnLongColumn { + + private static final long serialVersionUID = 1L; + + public IfExprDecimal64ColumnDecimal64Column(int arg1Column, int arg2Column, int arg3Column, + int outputColumnNum) { + super(arg1Column, arg2Column, arg3Column, outputColumnNum); + } + + public IfExprDecimal64ColumnDecimal64Column() { + super(); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(3) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("long"), + VectorExpressionDescriptor.ArgumentType.DECIMAL_64, + VectorExpressionDescriptor.ArgumentType.DECIMAL_64) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } +} \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprDecimal64ColumnDecimal64Scalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprDecimal64ColumnDecimal64Scalar.java new file mode 100644 index 0000000..f5f9dc9 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprDecimal64ColumnDecimal64Scalar.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongColumnLongScalar; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; + +/** + * Compute IF(expr1, expr2, expr3) for 3 input column expressions. + * The first is always a boolean (LongColumnVector). + * The second is a column or non-constant expression result. + * The third is a constant value. + */ +public class IfExprDecimal64ColumnDecimal64Scalar extends IfExprLongColumnLongScalar { + + private static final long serialVersionUID = 1L; + + public IfExprDecimal64ColumnDecimal64Scalar(int arg1Column, int arg2Column, long arg3Scalar, + int outputColumnNum) { + super(arg1Column, arg2Column, arg3Scalar, outputColumnNum); + } + + public IfExprDecimal64ColumnDecimal64Scalar() { + super(); + } + + @Override + public String vectorExpressionParameters() { + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) inputTypeInfos[2]; + HiveDecimalWritable writable = new HiveDecimalWritable(); + writable.deserialize64(arg3Scalar, decimalTypeInfo.scale()); + return getColumnParamString(0, arg1Column) + ", " + getColumnParamString(1, arg2Column) + + ", decimal64Val " + arg3Scalar + + ", decimalVal " + writable.toString(); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(3) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("long"), + VectorExpressionDescriptor.ArgumentType.DECIMAL_64, + VectorExpressionDescriptor.ArgumentType.DECIMAL_64) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprDecimal64ScalarDecimal64Column.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprDecimal64ScalarDecimal64Column.java new file mode 100644 index 0000000..cf16c8a --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprDecimal64ScalarDecimal64Column.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongScalarLongColumn; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; + +/** + * Compute IF(expr1, expr2, expr3) for 3 input column expressions. + * The first is always a boolean (LongColumnVector). + * The second is a column or non-constant expression result. + * The third is a constant value. + */ +public class IfExprDecimal64ScalarDecimal64Column extends IfExprLongScalarLongColumn { + + private static final long serialVersionUID = 1L; + + public IfExprDecimal64ScalarDecimal64Column(int arg1Column, long arg2Scalar, int arg3Column, + int outputColumnNum) { + super(arg1Column, arg2Scalar, arg3Column, outputColumnNum); + } + + public IfExprDecimal64ScalarDecimal64Column() { + super(); + } + + @Override + public String vectorExpressionParameters() { + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) inputTypeInfos[1]; + HiveDecimalWritable writable = new HiveDecimalWritable(); + writable.deserialize64(arg2Scalar, decimalTypeInfo.scale()); + return + getColumnParamString(0, arg1Column) + + ", decimal64Val " + arg2Scalar + ", decimalVal " + writable.toString() + + ", " + getColumnParamString(1, arg3Column); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(3) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("long"), + VectorExpressionDescriptor.ArgumentType.DECIMAL_64, + VectorExpressionDescriptor.ArgumentType.DECIMAL_64) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprDecimal64ScalarDecimal64Scalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprDecimal64ScalarDecimal64Scalar.java new file mode 100644 index 0000000..8e76a94 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprDecimal64ScalarDecimal64Scalar.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongScalarLongScalar; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; + +/** + * Compute IF(expr1, expr2, expr3) for 3 input expressions. + * The first is always a boolean (LongColumnVector). + * The second is a constant value. + * The third is a constant value. + */ +public class IfExprDecimal64ScalarDecimal64Scalar extends IfExprLongScalarLongScalar { + + private static final long serialVersionUID = 1L; + + public IfExprDecimal64ScalarDecimal64Scalar(int arg1Column, long arg2Scalar, long arg3Scalar, + int outputColumnNum) { + super(arg1Column, arg2Scalar, arg3Scalar, outputColumnNum); + } + + public IfExprDecimal64ScalarDecimal64Scalar() { + super(); + } + + @Override + public String vectorExpressionParameters() { + DecimalTypeInfo decimalTypeInfo1 = (DecimalTypeInfo) inputTypeInfos[1]; + HiveDecimalWritable writable1 = new HiveDecimalWritable(); + writable1.deserialize64(arg2Scalar, decimalTypeInfo1.scale()); + + DecimalTypeInfo decimalTypeInfo2 = (DecimalTypeInfo) inputTypeInfos[2]; + HiveDecimalWritable writable2 = new HiveDecimalWritable(); + writable2.deserialize64(arg3Scalar, decimalTypeInfo2.scale()); + return + getColumnParamString(0, arg1Column) + + ", decimal64Val1 " + arg2Scalar + ", decimalVal1 " + writable1.toString() + + ", decimal64Val2 " + arg3Scalar + ", decimalVal2 " + writable2.toString(); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(3) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("long"), + VectorExpressionDescriptor.ArgumentType.DECIMAL_64, + VectorExpressionDescriptor.ArgumentType.DECIMAL_64) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR, + VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprLongColumnLongColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprLongColumnLongColumn.java index 2a10e29..e4cb57b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprLongColumnLongColumn.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprLongColumnLongColumn.java @@ -31,9 +31,9 @@ private static final long serialVersionUID = 1L; - private final int arg1Column; - private final int arg2Column; - private final int arg3Column; + protected final int arg1Column; + protected final int arg2Column; + protected final int arg3Column; public IfExprLongColumnLongColumn(int arg1Column, int arg2Column, int arg3Column, int outputColumnNum) { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringGroupColumnStringGroupColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringGroupColumnStringGroupColumn.java index fc244ec..6290858 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringGroupColumnStringGroupColumn.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringGroupColumnStringGroupColumn.java @@ -196,8 +196,8 @@ public String vectorExpressionParameters() { .setNumArguments(3) .setArgumentTypes( VectorExpressionDescriptor.ArgumentType.INT_FAMILY, - VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, - VectorExpressionDescriptor.ArgumentType.STRING_FAMILY) + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY_BINARY, + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY_BINARY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.COLUMN, diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringGroupColumnStringScalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringGroupColumnStringScalar.java index e6b30d9..728f955 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringGroupColumnStringScalar.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringGroupColumnStringScalar.java @@ -185,8 +185,8 @@ public String vectorExpressionParameters() { .setNumArguments(3) .setArgumentTypes( VectorExpressionDescriptor.ArgumentType.INT_FAMILY, - VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, - VectorExpressionDescriptor.ArgumentType.STRING) + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY_BINARY, + VectorExpressionDescriptor.ArgumentType.STRING_BINARY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.COLUMN, diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringGroupColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringGroupColumn.java index 52ed087..de852e7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringGroupColumn.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringGroupColumn.java @@ -188,8 +188,8 @@ public String vectorExpressionParameters() { .setNumArguments(3) .setArgumentTypes( VectorExpressionDescriptor.ArgumentType.INT_FAMILY, - VectorExpressionDescriptor.ArgumentType.STRING, - VectorExpressionDescriptor.ArgumentType.STRING_FAMILY) + VectorExpressionDescriptor.ArgumentType.STRING_BINARY, + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY_BINARY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.SCALAR, diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringScalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringScalar.java index 9992743..05773a4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringScalar.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringScalar.java @@ -152,8 +152,8 @@ public String vectorExpressionParameters() { .setNumArguments(3) .setArgumentTypes( VectorExpressionDescriptor.ArgumentType.INT_FAMILY, - VectorExpressionDescriptor.ArgumentType.STRING, - VectorExpressionDescriptor.ArgumentType.STRING) + VectorExpressionDescriptor.ArgumentType.STRING_BINARY, + VectorExpressionDescriptor.ArgumentType.STRING_BINARY) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.SCALAR, diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprVarCharScalarStringScalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprVarCharScalarStringScalar.java index 6ab1ad7..814e3fe 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprVarCharScalarStringScalar.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprVarCharScalarStringScalar.java @@ -47,7 +47,7 @@ public IfExprVarCharScalarStringScalar() { .setArgumentTypes( VectorExpressionDescriptor.ArgumentType.INT_FAMILY, VectorExpressionDescriptor.ArgumentType.VARCHAR, - VectorExpressionDescriptor.ArgumentType.STRING) + VectorExpressionDescriptor.ArgumentType.VARCHAR) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.SCALAR, diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColEqualLongColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColEqualLongColumn.java deleted file mode 100644 index 308ddf9..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColEqualLongColumn.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongColEqualLongColumn extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private final int colNum1; - private final int colNum2; - - public LongColEqualLongColumn(int colNum1, int colNum2, int outputColumnNum) { - super(outputColumnNum); - this.colNum1 = colNum1; - this.colNum2 = colNum2; - } - - public LongColEqualLongColumn() { - super(); - - // Dummy final assignments. - colNum1 = -1; - colNum2 = -1; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1]; - LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - int n = batch.size; - long[] vector1 = inputColVector1.vector; - long[] vector2 = inputColVector2.vector; - long[] outputVector = outputColVector.vector; - long vector1Value = vector1[0]; - long vector2Value = vector2[0]; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - /* - * Propagate null values for a two-input operator and set isRepeating and noNulls appropriately. - */ - NullUtil.propagateNullsColCol( - inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); - - /* Disregard nulls for processing. In other words, - * the arithmetic operation is performed even if one or - * more inputs are null. This is to improve speed by avoiding - * conditional checks in the inner loop. - */ - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - outputVector[0] = vector1Value == vector2Value ? 1 : 0; - } else if (inputColVector1.isRepeating) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1Value == vector2[i] ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - // The SIMD optimized form of "a == b" is "(((a - b) ^ (b - a)) >>> 63) ^ 1" - outputVector[i] = (((vector1Value - vector2[i]) ^ (vector2[i] - vector1Value)) >>> 63) ^ 1; - } - } - } else if (inputColVector2.isRepeating) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1[i] == vector2Value ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = (((vector1[i] - vector2Value) ^ (vector2Value - vector1[i])) >>> 63) ^ 1; - } - } - } else { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1[i] == vector2[i] ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = (((vector1[i] - vector2[i]) ^ (vector2[i] - vector1[i])) >>> 63) ^ 1; - } - } - } - - /* For the case when the output can have null values, follow - * the convention that the data values must be 1 for long and - * NaN for double. This is to prevent possible later zero-divide errors - * in complex arithmetic expressions like col2 / (col1 - 1) - * in the case when some col1 entries are null. - */ - NullUtil.setNullDataEntriesLong(outputColVector, batch.selectedInUse, sel, n); - } - - @Override - public String vectorExpressionParameters() { - return getColumnParamString(0, colNum1) + ", " + getColumnParamString(1, colNum2); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("int_date_interval_year_month"), - VectorExpressionDescriptor.ArgumentType.getType("int_date_interval_year_month")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColEqualLongScalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColEqualLongScalar.java deleted file mode 100644 index 797dd39..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColEqualLongScalar.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import java.util.Arrays; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongColEqualLongScalar extends VectorExpression { - private static final long serialVersionUID = 1L; - - protected final int colNum; - protected final long value; - - public LongColEqualLongScalar(int colNum, long value, int outputColumnNum) { - super(outputColumnNum); - this.colNum = colNum; - this.value = value; - } - - public LongColEqualLongScalar() { - super(); - - // Dummy final assignments. - colNum = -1; - value = 0; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - // return immediately if batch is empty - final int n = batch.size; - if (n == 0) { - return; - } - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; - boolean[] outputIsNull = outputColVector.isNull; - - long[] vector = inputColVector.vector; - long[] outputVector = outputColVector.vector; - - // We do not need to do a column reset since we are carefully changing the output. - outputColVector.isRepeating = false; - - if (inputColVector.isRepeating) { - if (inputColVector.noNulls || !inputIsNull[0]) { - outputIsNull[0] = false; - outputVector[0] = vector[0] == value ? 1 : 0; - } else { - outputIsNull[0] = true; - outputColVector.noNulls = false; - } - outputColVector.isRepeating = true; - return; - } - - if (inputColVector.noNulls) { - if (batch.selectedInUse) { - - // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop. - - if (!outputColVector.noNulls) { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - // Set isNull before call in case it changes it mind. - outputIsNull[i] = false; - outputVector[i] = (((vector[i] - value) ^ (value - vector[i])) >>> 63) ^ 1; - } - } else { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - outputVector[i] = (((vector[i] - value) ^ (value - vector[i])) >>> 63) ^ 1; - } - } - } else { - if (!outputColVector.noNulls) { - - // Assume it is almost always a performance win to fill all of isNull so we can - // safely reset noNulls. - Arrays.fill(outputIsNull, false); - outputColVector.noNulls = true; - } - for(int i = 0; i != n; i++) { - outputVector[i] = (((vector[i] - value) ^ (value - vector[i])) >>> 63) ^ 1; - } - } - } else /* there are nulls in the inputColVector */ { - - // Carefully handle NULLs... - - /* - * For better performance on LONG/DOUBLE we don't want the conditional - * statements inside the for loop. - */ - outputColVector.noNulls = false; - - if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputIsNull[i] = inputIsNull[i]; - outputVector[i] = (((vector[i] - value) ^ (value - vector[i])) >>> 63) ^ 1; - } - } else { - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - for(int i = 0; i != n; i++) { - outputVector[i] = (((vector[i] - value) ^ (value - vector[i])) >>> 63) ^ 1; - } - } - } - } - - public String vectorExpressionParameters() { - return getColumnParamString(0, colNum) + ", val " + value; - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("long"), - VectorExpressionDescriptor.ArgumentType.getType("long")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColGreaterEqualLongColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColGreaterEqualLongColumn.java deleted file mode 100644 index ad9c7a3..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColGreaterEqualLongColumn.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongColGreaterEqualLongColumn extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private final int colNum1; - private final int colNum2; - - public LongColGreaterEqualLongColumn(int colNum1, int colNum2, int outputColumnNum) { - super(outputColumnNum); - this.colNum1 = colNum1; - this.colNum2 = colNum2; - } - - public LongColGreaterEqualLongColumn() { - super(); - - // Dummy final assignments. - colNum1 = -1; - colNum2 = -1; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1]; - LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - int n = batch.size; - long[] vector1 = inputColVector1.vector; - long[] vector2 = inputColVector2.vector; - long[] outputVector = outputColVector.vector; - long vector1Value = vector1[0]; - long vector2Value = vector2[0]; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - /* - * Propagate null values for a two-input operator and set isRepeating and noNulls appropriately. - */ - NullUtil.propagateNullsColCol( - inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); - - /* Disregard nulls for processing. In other words, - * the arithmetic operation is performed even if one or - * more inputs are null. This is to improve speed by avoiding - * conditional checks in the inner loop. - */ - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - outputVector[0] = vector1Value >= vector2Value ? 1 : 0; - } else if (inputColVector1.isRepeating) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1Value >= vector2[i] ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - // The SIMD optimized form of "a >= b" is "((a - b) >>> 63) ^ 1" - outputVector[i] = ((vector1Value - vector2[i]) >>> 63) ^ 1; - } - } - } else if (inputColVector2.isRepeating) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1[i] >= vector2Value ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = ((vector1[i] - vector2Value) >>> 63) ^ 1; - } - } - } else { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1[i] >= vector2[i] ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = ((vector1[i] - vector2[i]) >>> 63) ^ 1; - } - } - } - - /* For the case when the output can have null values, follow - * the convention that the data values must be 1 for long and - * NaN for double. This is to prevent possible later zero-divide errors - * in complex arithmetic expressions like col2 / (col1 - 1) - * in the case when some col1 entries are null. - */ - NullUtil.setNullDataEntriesLong(outputColVector, batch.selectedInUse, sel, n); - } - - @Override - public String vectorExpressionParameters() { - return getColumnParamString(0, colNum1) + ", " + getColumnParamString(1, colNum2); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("int_date_interval_year_month"), - VectorExpressionDescriptor.ArgumentType.getType("int_date_interval_year_month")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColGreaterEqualLongScalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColGreaterEqualLongScalar.java deleted file mode 100644 index 497826a..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColGreaterEqualLongScalar.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import java.util.Arrays; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongColGreaterEqualLongScalar extends VectorExpression { - - private static final long serialVersionUID = 1L; - - protected int colNum; - protected long value; - - public LongColGreaterEqualLongScalar(int colNum, long value, int outputColumnNum) { - super(outputColumnNum); - this.colNum = colNum; - this.value = value; - } - - public LongColGreaterEqualLongScalar() { - super(); - - // Dummy final assignments. - colNum = -1; - value = 0; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; - boolean[] outputIsNull = outputColVector.isNull; - int n = batch.size; - long[] vector = inputColVector.vector; - long[] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - // We do not need to do a column reset since we are carefully changing the output. - outputColVector.isRepeating = false; - - if (inputColVector.isRepeating) { - if (inputColVector.noNulls || !inputIsNull[0]) { - outputIsNull[0] = false; - outputVector[0] = vector[0] >= value ? 1 : 0; - } else { - outputIsNull[0] = true; - outputColVector.noNulls = false; - } - outputColVector.isRepeating = true; - return; - } - - if (inputColVector.noNulls) { - if (batch.selectedInUse) { - - // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop. - - if (!outputColVector.noNulls) { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - // Set isNull before call in case it changes it mind. - outputIsNull[i] = false; - outputVector[i] = ((vector[i] - value) >>> 63) ^ 1; - } - } else { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - outputVector[i] = ((vector[i] - value) >>> 63) ^ 1; - } - } - } else { - if (!outputColVector.noNulls) { - - // Assume it is almost always a performance win to fill all of isNull so we can - // safely reset noNulls. - Arrays.fill(outputIsNull, false); - outputColVector.noNulls = true; - } - for(int i = 0; i != n; i++) { - outputVector[i] = ((vector[i] - value) >>> 63) ^ 1; - } - } - } else /* there are nulls in the inputColVector */ { - - // Carefully handle NULLs... - - /* - * For better performance on LONG/DOUBLE we don't want the conditional - * statements inside the for loop. - */ - outputColVector.noNulls = false; - - if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputIsNull[i] = inputIsNull[i]; - outputVector[i] = ((vector[i] - value) >>> 63) ^ 1; - } - } else { - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - for(int i = 0; i != n; i++) { - outputVector[i] = ((vector[i] - value) >>> 63) ^ 1; - } - } - } - } - - @Override - public String vectorExpressionParameters() { - return getColumnParamString(0, colNum) + ", val " + value; - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("long"), - VectorExpressionDescriptor.ArgumentType.getType("long")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColGreaterLongColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColGreaterLongColumn.java deleted file mode 100644 index 3e1a13e..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColGreaterLongColumn.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongColGreaterLongColumn extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private final int colNum1; - private final int colNum2; - - public LongColGreaterLongColumn(int colNum1, int colNum2, int outputColumnNum) { - super(outputColumnNum); - this.colNum1 = colNum1; - this.colNum2 = colNum2; - } - - public LongColGreaterLongColumn() { - super(); - - // Dummy final assignments. - colNum1 = -1; - colNum2 = -1; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1]; - LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - int n = batch.size; - long[] vector1 = inputColVector1.vector; - long[] vector2 = inputColVector2.vector; - long[] outputVector = outputColVector.vector; - long vector1Value = vector1[0]; - long vector2Value = vector2[0]; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - /* - * Propagate null values for a two-input operator and set isRepeating and noNulls appropriately. - */ - NullUtil.propagateNullsColCol( - inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); - - /* Disregard nulls for processing. In other words, - * the arithmetic operation is performed even if one or - * more inputs are null. This is to improve speed by avoiding - * conditional checks in the inner loop. - */ - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - outputVector[0] = vector1Value > vector2Value ? 1 : 0; - } else if (inputColVector1.isRepeating) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1Value > vector2[i] ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - // The SIMD optimized form of "a > b" is "(b - a) >>> 63" - outputVector[i] = (vector2[i] - vector1Value) >>> 63; - } - } - } else if (inputColVector2.isRepeating) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1[i] > vector2Value ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = (vector2Value - vector1[i]) >>> 63; - } - } - } else { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1[i] > vector2[i] ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = (vector2[i] - vector1[i]) >>> 63; - } - } - } - - /* For the case when the output can have null values, follow - * the convention that the data values must be 1 for long and - * NaN for double. This is to prevent possible later zero-divide errors - * in complex arithmetic expressions like col2 / (col1 - 1) - * in the case when some col1 entries are null. - */ - NullUtil.setNullDataEntriesLong(outputColVector, batch.selectedInUse, sel, n); - } - - @Override - public String vectorExpressionParameters() { - return getColumnParamString(0, colNum1) + ", " + getColumnParamString(1, colNum2); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("int_date_interval_year_month"), - VectorExpressionDescriptor.ArgumentType.getType("int_date_interval_year_month")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColGreaterLongScalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColGreaterLongScalar.java deleted file mode 100644 index a4deedf..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColGreaterLongScalar.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import java.util.Arrays; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongColGreaterLongScalar extends VectorExpression { - private static final long serialVersionUID = 1L; - - protected final int colNum; - protected final long value; - - public LongColGreaterLongScalar(int colNum, long value, int outputColumnNum) { - super(outputColumnNum); - this.colNum = colNum; - this.value = value; - } - - public LongColGreaterLongScalar() { - super(); - - // Dummy final assignments. - colNum = -1; - value = 0; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; - boolean[] outputIsNull = outputColVector.isNull; - int n = batch.size; - long[] vector = inputColVector.vector; - long[] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - // We do not need to do a column reset since we are carefully changing the output. - outputColVector.isRepeating = false; - - if (inputColVector.isRepeating) { - if (inputColVector.noNulls || !inputIsNull[0]) { - outputIsNull[0] = false; - outputVector[0] = vector[0] > value ? 1 : 0; - } else { - outputIsNull[0] = true; - outputColVector.noNulls = false; - } - outputColVector.isRepeating = true; - return; - } - - if (inputColVector.noNulls) { - if (batch.selectedInUse) { - - // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop. - - if (!outputColVector.noNulls) { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - // Set isNull before call in case it changes it mind. - outputIsNull[i] = false; - outputVector[i] = (value - vector[i]) >>> 63; - } - } else { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - outputVector[i] = (value - vector[i]) >>> 63; - } - } - } else { - if (!outputColVector.noNulls) { - - // Assume it is almost always a performance win to fill all of isNull so we can - // safely reset noNulls. - Arrays.fill(outputIsNull, false); - outputColVector.noNulls = true; - } - for(int i = 0; i != n; i++) { - outputVector[i] = (value - vector[i]) >>> 63; - } - } - } else /* there are nulls in the inputColVector */ { - - // Carefully handle NULLs... - - /* - * For better performance on LONG/DOUBLE we don't want the conditional - * statements inside the for loop. - */ - outputColVector.noNulls = false; - - if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputIsNull[i] = inputIsNull[i]; - outputVector[i] = (value - vector[i]) >>> 63; - } - } else { - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - for(int i = 0; i != n; i++) { - outputVector[i] = (value - vector[i]) >>> 63; - } - } - } - } - - @Override - public String vectorExpressionParameters() { - return getColumnParamString(0, colNum) + ", val " + value; - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("long"), - VectorExpressionDescriptor.ArgumentType.getType("long")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColLessEqualLongColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColLessEqualLongColumn.java deleted file mode 100644 index 3212269..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColLessEqualLongColumn.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongColLessEqualLongColumn extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private final int colNum1; - private final int colNum2; - - public LongColLessEqualLongColumn(int colNum1, int colNum2, int outputColumnNum) { - super(outputColumnNum); - this.colNum1 = colNum1; - this.colNum2 = colNum2; - } - - public LongColLessEqualLongColumn() { - super(); - - // Dummy final assignments. - colNum1 = -1; - colNum2 = -1; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1]; - LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - int n = batch.size; - long[] vector1 = inputColVector1.vector; - long[] vector2 = inputColVector2.vector; - long[] outputVector = outputColVector.vector; - long vector1Value = vector1[0]; - long vector2Value = vector2[0]; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - /* - * Propagate null values for a two-input operator and set isRepeating and noNulls appropriately. - */ - NullUtil.propagateNullsColCol( - inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); - - /* Disregard nulls for processing. In other words, - * the arithmetic operation is performed even if one or - * more inputs are null. This is to improve speed by avoiding - * conditional checks in the inner loop. - */ - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - outputVector[0] = vector1Value <= vector2Value ? 1 : 0; - } else if (inputColVector1.isRepeating) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1Value <= vector2[i] ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - // The SIMD optimized form of "a <= b" is "((b - a) >>> 63) ^ 1" - outputVector[i] = ((vector2[i] - vector1Value) >>> 63) ^ 1; - } - } - } else if (inputColVector2.isRepeating) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1[i] <= vector2Value ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = ((vector2Value - vector1[i]) >>> 63) ^ 1; - } - } - } else { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1[i] <= vector2[i] ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = ((vector2[i] - vector1[i]) >>> 63) ^ 1; - } - } - } - - /* For the case when the output can have null values, follow - * the convention that the data values must be 1 for long and - * NaN for double. This is to prevent possible later zero-divide errors - * in complex arithmetic expressions like col2 / (col1 - 1) - * in the case when some col1 entries are null. - */ - NullUtil.setNullDataEntriesLong(outputColVector, batch.selectedInUse, sel, n); - } - - @Override - public String vectorExpressionParameters() { - return getColumnParamString(0, colNum1) + ", " + getColumnParamString(1, colNum2); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("int_date_interval_year_month"), - VectorExpressionDescriptor.ArgumentType.getType("int_date_interval_year_month")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColLessEqualLongScalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColLessEqualLongScalar.java deleted file mode 100644 index c2f0de1..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColLessEqualLongScalar.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import java.util.Arrays; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongColLessEqualLongScalar extends VectorExpression { - - private static final long serialVersionUID = 1L; - - protected int colNum; - protected long value; - - public LongColLessEqualLongScalar(int colNum, long value, int outputColumnNum) { - super(outputColumnNum); - this.colNum = colNum; - this.value = value; - } - - public LongColLessEqualLongScalar() { - super(); - - // Dummy final assignments. - colNum = -1; - value = 0; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; - boolean[] outputIsNull = outputColVector.isNull; - int n = batch.size; - long[] vector = inputColVector.vector; - long[] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - // We do not need to do a column reset since we are carefully changing the output. - outputColVector.isRepeating = false; - - if (inputColVector.isRepeating) { - if (inputColVector.noNulls || !inputIsNull[0]) { - outputIsNull[0] = false; - outputVector[0] = vector[0] <= value ? 1 : 0; - } else { - outputIsNull[0] = true; - outputColVector.noNulls = false; - } - outputColVector.isRepeating = true; - return; - } - - if (inputColVector.noNulls) { - if (batch.selectedInUse) { - - // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop. - - if (!outputColVector.noNulls) { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - // Set isNull before call in case it changes it mind. - outputIsNull[i] = false; - outputVector[i] = ((value - vector[i]) >>> 63) ^ 1; - } - } else { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - outputVector[i] = ((value - vector[i]) >>> 63) ^ 1; - } - } - } else { - if (!outputColVector.noNulls) { - - // Assume it is almost always a performance win to fill all of isNull so we can - // safely reset noNulls. - Arrays.fill(outputIsNull, false); - outputColVector.noNulls = true; - } - for(int i = 0; i != n; i++) { - outputVector[i] = ((value - vector[i]) >>> 63) ^ 1; - } - } - } else /* there are nulls in the inputColVector */ { - - // Carefully handle NULLs... - - /* - * For better performance on LONG/DOUBLE we don't want the conditional - * statements inside the for loop. - */ - outputColVector.noNulls = false; - - if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputIsNull[i] = inputIsNull[i]; - outputVector[i] = ((value - vector[i]) >>> 63) ^ 1; - } - } else { - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - for(int i = 0; i != n; i++) { - outputVector[i] = ((value - vector[i]) >>> 63) ^ 1; - } - } - } - } - - @Override - public String vectorExpressionParameters() { - return getColumnParamString(0, colNum) + ", val " + value; - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("long"), - VectorExpressionDescriptor.ArgumentType.getType("long")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColLessLongColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColLessLongColumn.java deleted file mode 100644 index 7c2ee20..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColLessLongColumn.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongColLessLongColumn extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private final int colNum1; - private final int colNum2; - - public LongColLessLongColumn(int colNum1, int colNum2, int outputColumnNum) { - super(outputColumnNum); - this.colNum1 = colNum1; - this.colNum2 = colNum2; - } - - public LongColLessLongColumn() { - super(); - - // Dummy final assignments. - colNum1 = -1; - colNum2 = -1; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1]; - LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - int n = batch.size; - long[] vector1 = inputColVector1.vector; - long[] vector2 = inputColVector2.vector; - long[] outputVector = outputColVector.vector; - long vector1Value = vector1[0]; - long vector2Value = vector2[0]; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - /* - * Propagate null values for a two-input operator and set isRepeating and noNulls appropriately. - */ - NullUtil.propagateNullsColCol( - inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); - - /* Disregard nulls for processing. In other words, - * the arithmetic operation is performed even if one or - * more inputs are null. This is to improve speed by avoiding - * conditional checks in the inner loop. - */ - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - outputVector[0] = vector1Value < vector2Value ? 1 : 0; - } else if (inputColVector1.isRepeating) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1Value < vector2[i] ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - // The SIMD optimized form of "a < b" is "(a - b) >>> 63" - outputVector[i] = (vector1Value - vector2[i]) >>> 63; - } - } - } else if (inputColVector2.isRepeating) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1[i] < vector2Value ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = (vector1[i] - vector2Value) >>> 63; - } - } - } else { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1[i] < vector2[i] ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = (vector1[i] - vector2[i]) >>> 63; - } - } - } - - /* For the case when the output can have null values, follow - * the convention that the data values must be 1 for long and - * NaN for double. This is to prevent possible later zero-divide errors - * in complex arithmetic expressions like col2 / (col1 - 1) - * in the case when some col1 entries are null. - */ - NullUtil.setNullDataEntriesLong(outputColVector, batch.selectedInUse, sel, n); - } - - @Override - public String vectorExpressionParameters() { - return getColumnParamString(0, colNum1) + ", " + getColumnParamString(1, colNum2); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("int_date_interval_year_month"), - VectorExpressionDescriptor.ArgumentType.getType("int_date_interval_year_month")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColLessLongScalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColLessLongScalar.java deleted file mode 100644 index c7efe84..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColLessLongScalar.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import java.util.Arrays; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongColLessLongScalar extends VectorExpression { - - private static final long serialVersionUID = 1L; - - protected final int colNum; - protected final long value; - - public LongColLessLongScalar(int colNum, long value, int outputColumnNum) { - super(outputColumnNum); - this.colNum = colNum; - this.value = value; - } - - public LongColLessLongScalar() { - super(); - - // Dummy final assignments. - colNum = -1; - value = 0; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; - boolean[] outputIsNull = outputColVector.isNull; - int n = batch.size; - long[] vector = inputColVector.vector; - long[] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - // We do not need to do a column reset since we are carefully changing the output. - outputColVector.isRepeating = false; - - if (inputColVector.isRepeating) { - if (inputColVector.noNulls || !inputIsNull[0]) { - outputIsNull[0] = false; - outputVector[0] = vector[0] < value ? 1 : 0; - } else { - outputIsNull[0] = true; - outputColVector.noNulls = false; - } - outputColVector.isRepeating = true; - return; - } - - if (inputColVector.noNulls) { - if (batch.selectedInUse) { - - // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop. - - if (!outputColVector.noNulls) { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - // Set isNull before call in case it changes it mind. - outputIsNull[i] = false; - outputVector[i] = (vector[i] - value) >>> 63; - } - } else { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - outputVector[i] = (vector[i] - value) >>> 63; - } - } - } else { - if (!outputColVector.noNulls) { - - // Assume it is almost always a performance win to fill all of isNull so we can - // safely reset noNulls. - Arrays.fill(outputIsNull, false); - outputColVector.noNulls = true; - } - for(int i = 0; i != n; i++) { - outputVector[i] = (vector[i] - value) >>> 63; - } - } - } else /* there are nulls in the inputColVector */ { - - // Carefully handle NULLs... - - /* - * For better performance on LONG/DOUBLE we don't want the conditional - * statements inside the for loop. - */ - outputColVector.noNulls = false; - - if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputIsNull[i] = inputIsNull[i]; - outputVector[i] = (vector[i] - value) >>> 63; - } - } else { - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - for(int i = 0; i != n; i++) { - outputVector[i] = (vector[i] - value) >>> 63; - } - } - } - } - - @Override - public String vectorExpressionParameters() { - return getColumnParamString(0, colNum) + ", val " + value; - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("long"), - VectorExpressionDescriptor.ArgumentType.getType("long")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColNotEqualLongColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColNotEqualLongColumn.java deleted file mode 100644 index 213b876..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColNotEqualLongColumn.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongColNotEqualLongColumn extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private final int colNum1; - private final int colNum2; - - public LongColNotEqualLongColumn(int colNum1, int colNum2, int outputColumnNum) { - super(outputColumnNum); - this.colNum1 = colNum1; - this.colNum2 = colNum2; - } - - public LongColNotEqualLongColumn() { - super(); - - // Dummy final assignments. - colNum1 = -1; - colNum2 = -1; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1]; - LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - int n = batch.size; - long[] vector1 = inputColVector1.vector; - long[] vector2 = inputColVector2.vector; - long[] outputVector = outputColVector.vector; - long vector1Value = vector1[0]; - long vector2Value = vector2[0]; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - /* - * Propagate null values for a two-input operator and set isRepeating and noNulls appropriately. - */ - NullUtil.propagateNullsColCol( - inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); - - /* Disregard nulls for processing. In other words, - * the arithmetic operation is performed even if one or - * more inputs are null. This is to improve speed by avoiding - * conditional checks in the inner loop. - */ - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - outputVector[0] = vector1Value != vector2Value ? 1 : 0; - } else if (inputColVector1.isRepeating) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1Value != vector2[i] ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - // The SIMD optimized form of "a != b" is "((a - b) ^ (b - a)) >>> 63" - outputVector[i] = ((vector1Value - vector2[i]) ^ (vector2[i] - vector1Value)) >>> 63; - } - } - } else if (inputColVector2.isRepeating) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1[i] != vector2Value ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = ((vector1[i] - vector2Value) ^ (vector2Value - vector1[i])) >>> 63; - } - } - } else { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1[i] != vector2[i] ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = ((vector1[i] - vector2[i]) ^ (vector2[i] - vector1[i])) >>> 63; - } - } - } - - /* For the case when the output can have null values, follow - * the convention that the data values must be 1 for long and - * NaN for double. This is to prevent possible later zero-divide errors - * in complex arithmetic expressions like col2 / (col1 - 1) - * in the case when some col1 entries are null. - */ - NullUtil.setNullDataEntriesLong(outputColVector, batch.selectedInUse, sel, n); - } - - @Override - public String vectorExpressionParameters() { - return getColumnParamString(0, colNum1) + ", " + getColumnParamString(1, colNum2); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("int_date_interval_year_month"), - VectorExpressionDescriptor.ArgumentType.getType("int_date_interval_year_month")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColNotEqualLongScalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColNotEqualLongScalar.java deleted file mode 100644 index c2b52b8..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColNotEqualLongScalar.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import java.util.Arrays; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongColNotEqualLongScalar extends VectorExpression { - - private static final long serialVersionUID = 1L; - - protected final int colNum; - protected final long value; - - public LongColNotEqualLongScalar(int colNum, long value, int outputColumnNum) { - super(outputColumnNum); - this.colNum = colNum; - this.value = value; - } - - public LongColNotEqualLongScalar() { - super(); - - // Dummy final assignments. - colNum = -1; - value = 0; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; - boolean[] outputIsNull = outputColVector.isNull; - int n = batch.size; - long[] vector = inputColVector.vector; - long[] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - // We do not need to do a column reset since we are carefully changing the output. - outputColVector.isRepeating = false; - - if (inputColVector.isRepeating) { - if (inputColVector.noNulls || !inputIsNull[0]) { - outputIsNull[0] = false; - outputVector[0] = vector[0] != value ? 1 : 0; - } else { - outputIsNull[0] = true; - outputColVector.noNulls = false; - } - outputColVector.isRepeating = true; - return; - } - - if (inputColVector.noNulls) { - if (batch.selectedInUse) { - - // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop. - - if (!outputColVector.noNulls) { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - // Set isNull before call in case it changes it mind. - outputIsNull[i] = false; - outputVector[i] = ((vector[i] - value) ^ (value - vector[i])) >>> 63; - } - } else { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - outputVector[i] = ((vector[i] - value) ^ (value - vector[i])) >>> 63; - } - } - } else { - if (!outputColVector.noNulls) { - - // Assume it is almost always a performance win to fill all of isNull so we can - // safely reset noNulls. - Arrays.fill(outputIsNull, false); - outputColVector.noNulls = true; - } - for(int i = 0; i != n; i++) { - outputVector[i] = ((vector[i] - value) ^ (value - vector[i])) >>> 63; - } - } - } else /* there are nulls in the inputColVector */ { - - // Carefully handle NULLs... - - /* - * For better performance on LONG/DOUBLE we don't want the conditional - * statements inside the for loop. - */ - outputColVector.noNulls = false; - - if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputIsNull[i] = inputIsNull[i]; - outputVector[i] = ((vector[i] - value) ^ (value - vector[i])) >>> 63; - } - } else { - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - for(int i = 0; i != n; i++) { - outputVector[i] = ((vector[i] - value) ^ (value - vector[i])) >>> 63; - } - } - } - } - - @Override - public String vectorExpressionParameters() { - return getColumnParamString(0, colNum) + ", val " + value; - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("long"), - VectorExpressionDescriptor.ArgumentType.getType("long")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarEqualLongColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarEqualLongColumn.java deleted file mode 100644 index 7b28f71..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarEqualLongColumn.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import java.util.Arrays; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongScalarEqualLongColumn extends VectorExpression { - - private static final long serialVersionUID = 1L; - - protected final int colNum; - protected final long value; - - public LongScalarEqualLongColumn(long value, int colNum, int outputColumnNum) { - super(outputColumnNum); - this.colNum = colNum; - this.value = value; - } - - public LongScalarEqualLongColumn() { - super(); - - // Dummy final assignments. - colNum = -1; - value = 0; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; - boolean[] outputIsNull = outputColVector.isNull; - int n = batch.size; - long[] vector = inputColVector.vector; - long[] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - // We do not need to do a column reset since we are carefully changing the output. - outputColVector.isRepeating = false; - - if (inputColVector.isRepeating) { - if (inputColVector.noNulls || !inputIsNull[0]) { - outputIsNull[0] = false; - outputVector[0] = vector[0] == value ? 1 : 0; - } else { - outputIsNull[0] = true; - outputColVector.noNulls = false; - } - outputColVector.isRepeating = true; - return; - } - - if (inputColVector.noNulls) { - if (batch.selectedInUse) { - - // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop. - - if (!outputColVector.noNulls) { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - // Set isNull before call in case it changes it mind. - outputIsNull[i] = false; - outputVector[i] = (((value - vector[i]) ^ (vector[i] - value)) >>> 63) ^ 1; - } - } else { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - outputVector[i] = (((value - vector[i]) ^ (vector[i] - value)) >>> 63) ^ 1; - } - } - } else { - if (!outputColVector.noNulls) { - - // Assume it is almost always a performance win to fill all of isNull so we can - // safely reset noNulls. - Arrays.fill(outputIsNull, false); - outputColVector.noNulls = true; - } - for(int i = 0; i != n; i++) { - outputVector[i] = (((value - vector[i]) ^ (vector[i] - value)) >>> 63) ^ 1; - } - } - } else /* there are nulls in the inputColVector */ { - - // Carefully handle NULLs... - - /* - * For better performance on LONG/DOUBLE we don't want the conditional - * statements inside the for loop. - */ - outputColVector.noNulls = false; - - if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputIsNull[i] = inputIsNull[i]; - outputVector[i] = (((value - vector[i]) ^ (vector[i] - value)) >>> 63) ^ 1; - } - } else { - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - for(int i = 0; i != n; i++) { - outputVector[i] = (((value - vector[i]) ^ (vector[i] - value)) >>> 63) ^ 1; - } - } - } - } - - public String vectorExpressionParameters() { - return "val " + value + ", " + getColumnParamString(1, colNum); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("long"), - VectorExpressionDescriptor.ArgumentType.getType("long")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.SCALAR, - VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarGreaterEqualLongColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarGreaterEqualLongColumn.java deleted file mode 100644 index 10fd423..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarGreaterEqualLongColumn.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import java.util.Arrays; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongScalarGreaterEqualLongColumn extends VectorExpression { - private static final long serialVersionUID = 1L; - - protected final int colNum; - protected final long value; - - public LongScalarGreaterEqualLongColumn(long value, int colNum, int outputColumnNum) { - super(outputColumnNum); - this.colNum = colNum; - this.value = value; - } - - public LongScalarGreaterEqualLongColumn() { - super(); - - // Dummy final assignments. - colNum = -1; - value = 0; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; - boolean[] outputIsNull = outputColVector.isNull; - int n = batch.size; - long[] vector = inputColVector.vector; - long[] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - // We do not need to do a column reset since we are carefully changing the output. - outputColVector.isRepeating = false; - - if (inputColVector.isRepeating) { - if (inputColVector.noNulls || !inputIsNull[0]) { - outputIsNull[0] = false; - outputVector[0] = value >= vector[0] ? 1 : 0; - } else { - outputIsNull[0] = true; - outputColVector.noNulls = false; - } - outputColVector.isRepeating = true; - return; - } - - if (inputColVector.noNulls) { - if (batch.selectedInUse) { - - // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop. - - if (!outputColVector.noNulls) { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - // Set isNull before call in case it changes it mind. - outputIsNull[i] = false; - // The SIMD optimized form of "a >= b" is "((a - b) >>> 63) ^ 1" - outputVector[i] = ((value - vector[i]) >>> 63) ^ 1; - } - } else { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - // The SIMD optimized form of "a >= b" is "((a - b) >>> 63) ^ 1" - outputVector[i] = ((value - vector[i]) >>> 63) ^ 1; - } - } - } else { - if (!outputColVector.noNulls) { - - // Assume it is almost always a performance win to fill all of isNull so we can - // safely reset noNulls. - Arrays.fill(outputIsNull, false); - outputColVector.noNulls = true; - } - for(int i = 0; i != n; i++) { - // The SIMD optimized form of "a >= b" is "((a - b) >>> 63) ^ 1" - outputVector[i] = ((value - vector[i]) >>> 63) ^ 1; - } - } - } else /* there are nulls in the inputColVector */ { - - // Carefully handle NULLs... - - /* - * For better performance on LONG/DOUBLE we don't want the conditional - * statements inside the for loop. - */ - outputColVector.noNulls = false; - - if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputIsNull[i] = inputIsNull[i]; - outputVector[i] = ((value - vector[i]) >>> 63) ^ 1; - } - } else { - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - for(int i = 0; i != n; i++) { - outputVector[i] = ((value - vector[i]) >>> 63) ^ 1; - } - } - } - } - - @Override - public String vectorExpressionParameters() { - return "val " + value + ", " + getColumnParamString(1, colNum); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("long"), - VectorExpressionDescriptor.ArgumentType.getType("long")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.SCALAR, - VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarGreaterLongColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarGreaterLongColumn.java deleted file mode 100644 index ad9ccf5..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarGreaterLongColumn.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import java.util.Arrays; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongScalarGreaterLongColumn extends VectorExpression { - - private static final long serialVersionUID = 1L; - - protected int colNum; - protected long value; - - public LongScalarGreaterLongColumn(long value, int colNum, int outputColumnNum) { - super(outputColumnNum); - this.colNum = colNum; - this.value = value; - } - - public LongScalarGreaterLongColumn() { - super(); - - // Dummy final assignments. - colNum = -1; - value = -1; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; - boolean[] outputIsNull = outputColVector.isNull; - int n = batch.size; - long[] vector = inputColVector.vector; - long[] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - // We do not need to do a column reset since we are carefully changing the output. - outputColVector.isRepeating = false; - - if (inputColVector.isRepeating) { - if (inputColVector.noNulls || !inputIsNull[0]) { - outputIsNull[0] = false; - outputVector[0] = value > vector[0] ? 1 : 0; - } else { - outputIsNull[0] = true; - outputColVector.noNulls = false; - } - outputColVector.isRepeating = true; - return; - } - - if (inputColVector.noNulls) { - if (batch.selectedInUse) { - - // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop. - - if (!outputColVector.noNulls) { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - // Set isNull before call in case it changes it mind. - outputIsNull[i] = false; - // The SIMD optimized form of "a > b" is "(b - a) >>> 63" - outputVector[i] = (vector[i] - value) >>> 63; - } - } else { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - // The SIMD optimized form of "a > b" is "(b - a) >>> 63" - outputVector[i] = (vector[i] - value) >>> 63; - } - } - } else { - if (!outputColVector.noNulls) { - - // Assume it is almost always a performance win to fill all of isNull so we can - // safely reset noNulls. - Arrays.fill(outputIsNull, false); - outputColVector.noNulls = true; - } - for(int i = 0; i != n; i++) { - // The SIMD optimized form of "a > b" is "(b - a) >>> 63" - outputVector[i] = (vector[i] - value) >>> 63; - } - } - } else /* there are nulls in the inputColVector */ { - - // Carefully handle NULLs... - - /* - * For better performance on LONG/DOUBLE we don't want the conditional - * statements inside the for loop. - */ - outputColVector.noNulls = false; - - if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputIsNull[i] = inputIsNull[i]; - outputVector[i] = (vector[i] - value) >>> 63; - } - } else { - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - for(int i = 0; i != n; i++) { - outputVector[i] = (vector[i] - value) >>> 63; - } - } - } - } - - @Override - public String vectorExpressionParameters() { - return "val " + value + ", " + getColumnParamString(0, colNum); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("long"), - VectorExpressionDescriptor.ArgumentType.getType("long")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.SCALAR, - VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarLessEqualLongColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarLessEqualLongColumn.java deleted file mode 100644 index abe5d08..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarLessEqualLongColumn.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import java.util.Arrays; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongScalarLessEqualLongColumn extends VectorExpression { - - private static final long serialVersionUID = 1L; - - protected final int colNum; - protected final long value; - - public LongScalarLessEqualLongColumn(long value, int colNum, int outputColumnNum) { - super(outputColumnNum); - this.colNum = colNum; - this.value = value; - } - - public LongScalarLessEqualLongColumn() { - super(); - - // Dummy final assignments. - colNum = -1; - value = 0; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; - boolean[] outputIsNull = outputColVector.isNull; - int n = batch.size; - long[] vector = inputColVector.vector; - long[] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - // We do not need to do a column reset since we are carefully changing the output. - outputColVector.isRepeating = false; - - if (inputColVector.isRepeating) { - if (inputColVector.noNulls || !inputIsNull[0]) { - outputIsNull[0] = false; - outputVector[0] = value <= vector[0] ? 1 : 0; - } else { - outputIsNull[0] = true; - outputColVector.noNulls = false; - } - outputColVector.isRepeating = true; - return; - } - - if (inputColVector.noNulls) { - if (batch.selectedInUse) { - - // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop. - - if (!outputColVector.noNulls) { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - // Set isNull before call in case it changes it mind. - outputIsNull[i] = false; - // The SIMD optimized form of "a <= b" is "((b - a) >>> 63) ^ 1" - outputVector[i] = ((vector[i] - value) >>> 63) ^ 1; - } - } else { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - // The SIMD optimized form of "a <= b" is "((b - a) >>> 63) ^ 1" - outputVector[i] = ((vector[i] - value) >>> 63) ^ 1; - } - } - } else { - if (!outputColVector.noNulls) { - - // Assume it is almost always a performance win to fill all of isNull so we can - // safely reset noNulls. - Arrays.fill(outputIsNull, false); - outputColVector.noNulls = true; - } - for(int i = 0; i != n; i++) { - // The SIMD optimized form of "a <= b" is "((b - a) >>> 63) ^ 1" - outputVector[i] = ((vector[i] - value) >>> 63) ^ 1; - } - } - } else /* there are nulls in the inputColVector */ { - - // Carefully handle NULLs... - - /* - * For better performance on LONG/DOUBLE we don't want the conditional - * statements inside the for loop. - */ - outputColVector.noNulls = false; - - if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputIsNull[i] = inputIsNull[i]; - outputVector[i] = ((vector[i] - value) >>> 63) ^ 1; - } - } else { - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - for(int i = 0; i != n; i++) { - outputVector[i] = ((vector[i] - value) >>> 63) ^ 1; - } - } - } - } - - public String vectorExpressionParameters() { - return "val " + value + ", " + getColumnParamString(1, colNum); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("long"), - VectorExpressionDescriptor.ArgumentType.getType("long")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.SCALAR, - VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarLessLongColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarLessLongColumn.java deleted file mode 100644 index 87ab939..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarLessLongColumn.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import java.util.Arrays; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongScalarLessLongColumn extends VectorExpression { - - private static final long serialVersionUID = 1L; - - protected final int colNum; - protected final long value; - - public LongScalarLessLongColumn(long value, int colNum, int outputColumnNum) { - super(outputColumnNum); - this.colNum = colNum; - this.value = value; - } - - public LongScalarLessLongColumn() { - super(); - - // Dummy final assignments. - colNum = -1; - value = 0; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; - boolean[] outputIsNull = outputColVector.isNull; - int n = batch.size; - long[] vector = inputColVector.vector; - long[] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - // We do not need to do a column reset since we are carefully changing the output. - outputColVector.isRepeating = false; - - if (inputColVector.isRepeating) { - if (inputColVector.noNulls || !inputIsNull[0]) { - outputIsNull[0] = false; - outputVector[0] = value < vector[0] ? 1 : 0; - } else { - outputIsNull[0] = true; - outputColVector.noNulls = false; - } - outputColVector.isRepeating = true; - return; - } - - if (inputColVector.noNulls) { - if (batch.selectedInUse) { - - // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop. - - if (!outputColVector.noNulls) { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - // Set isNull before call in case it changes it mind. - outputIsNull[i] = false; - // The SIMD optimized form of "a < b" is "(a - b) >>> 63" - outputVector[i] = (value - vector[i]) >>> 63; - } - } else { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - // The SIMD optimized form of "a < b" is "(a - b) >>> 63" - outputVector[i] = (value - vector[i]) >>> 63; - } - } - } else { - if (!outputColVector.noNulls) { - - // Assume it is almost always a performance win to fill all of isNull so we can - // safely reset noNulls. - Arrays.fill(outputIsNull, false); - outputColVector.noNulls = true; - } - for(int i = 0; i != n; i++) { - // The SIMD optimized form of "a < b" is "(a - b) >>> 63" - outputVector[i] = (value - vector[i]) >>> 63; - } - } - } else /* there are nulls in the inputColVector */ { - - // Carefully handle NULLs... - - /* - * For better performance on LONG/DOUBLE we don't want the conditional - * statements inside the for loop. - */ - outputColVector.noNulls = false; - - if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputIsNull[i] = inputIsNull[i]; - outputVector[i] = (value - vector[i]) >>> 63; - } - } else { - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - for(int i = 0; i != n; i++) { - outputVector[i] = (value - vector[i]) >>> 63; - } - } - } - } - - @Override - public String vectorExpressionParameters() { - return "val " + value + ", " + getColumnParamString(1, colNum); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("long"), - VectorExpressionDescriptor.ArgumentType.getType("long")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.SCALAR, - VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarNotEqualLongColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarNotEqualLongColumn.java deleted file mode 100644 index d936dee..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarNotEqualLongColumn.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import java.util.Arrays; - -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class LongScalarNotEqualLongColumn extends VectorExpression { - - private static final long serialVersionUID = 1L; - - protected final int colNum; - protected final long value; - - public LongScalarNotEqualLongColumn(long value, int colNum, int outputColumnNum) { - super(outputColumnNum); - this.colNum = colNum; - this.value = value; - } - - public LongScalarNotEqualLongColumn() { - super(); - - // Dummy final assignments. - colNum = -1; - value = 0; - } - - @Override - public void evaluate(VectorizedRowBatch batch) throws HiveException { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; - boolean[] outputIsNull = outputColVector.isNull; - int n = batch.size; - long[] vector = inputColVector.vector; - long[] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - // We do not need to do a column reset since we are carefully changing the output. - outputColVector.isRepeating = false; - - if (inputColVector.isRepeating) { - if (inputColVector.noNulls || !inputIsNull[0]) { - outputIsNull[0] = false; - outputVector[0] = value != vector[0] ? 1 : 0; - } else { - outputIsNull[0] = true; - outputColVector.noNulls = false; - } - outputColVector.isRepeating = true; - return; - } - - if (inputColVector.noNulls) { - if (batch.selectedInUse) { - - // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop. - - if (!outputColVector.noNulls) { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - // Set isNull before call in case it changes it mind. - outputIsNull[i] = false; - // The SIMD optimized form of "a != b" is "((a - b) ^ (b - a)) >>> 63" - outputVector[i] = ((vector[i] - value) ^ (value - vector[i])) >>> 63; - } - } else { - for(int j = 0; j != n; j++) { - final int i = sel[j]; - // The SIMD optimized form of "a != b" is "((a - b) ^ (b - a)) >>> 63" - outputVector[i] = ((vector[i] - value) ^ (value - vector[i])) >>> 63; - } - } - } else { - if (!outputColVector.noNulls) { - - // Assume it is almost always a performance win to fill all of isNull so we can - // safely reset noNulls. - Arrays.fill(outputIsNull, false); - outputColVector.noNulls = true; - } - for(int i = 0; i != n; i++) { - // The SIMD optimized form of "a != b" is "((a - b) ^ (b - a)) >>> 63" - outputVector[i] = ((vector[i] - value) ^ (value - vector[i])) >>> 63; - } - } - } else /* there are nulls in the inputColVector */ { - - // Carefully handle NULLs... - - /* - * For better performance on LONG/DOUBLE we don't want the conditional - * statements inside the for loop. - */ - outputColVector.noNulls = false; - - if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputIsNull[i] = inputIsNull[i]; - outputVector[i] = ((vector[i] - value) ^ (value - vector[i])) >>> 63; - } - } else { - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - for(int i = 0; i != n; i++) { - outputVector[i] = ((vector[i] - value) ^ (value - vector[i])) >>> 63; - } - } - } - } - - @Override - public String vectorExpressionParameters() { - return "val " + value + ", " + getColumnParamString(1, colNum); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("long"), - VectorExpressionDescriptor.ArgumentType.getType("long")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.SCALAR, - VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java index 142dd1b..25c54e9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java @@ -23,11 +23,16 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressionsSupportDecimal64; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDecimalColumnColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDecimalColumnScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDecimalScalarColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDecimalScalarScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongColumnLongScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleColumnDoubleScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleColumnLongScalar; @@ -45,6 +50,10 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprIntervalDayTimeColumnScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprIntervalDayTimeScalarColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprIntervalDayTimeScalarScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprDecimal64ColumnDecimal64Column; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprDecimal64ColumnDecimal64Scalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprDecimal64ScalarDecimal64Column; +import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprDecimal64ScalarDecimal64Scalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprLongColumnLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnStringGroupColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringGroupColumnStringScalar; @@ -85,6 +94,9 @@ IfExprLongScalarLongScalar.class, IfExprDoubleScalarDoubleScalar.class, IfExprLongScalarDoubleScalar.class, IfExprDoubleScalarLongScalar.class, + IfExprDecimal64ColumnDecimal64Column.class, IfExprDecimal64ColumnDecimal64Scalar.class, + IfExprDecimal64ScalarDecimal64Column.class, IfExprDecimal64ScalarDecimal64Scalar.class, + IfExprStringGroupColumnStringGroupColumn.class, IfExprStringGroupColumnStringScalar.class, IfExprStringGroupColumnCharScalar.class, IfExprStringGroupColumnVarCharScalar.class, @@ -94,11 +106,15 @@ IfExprStringScalarCharScalar.class, IfExprStringScalarVarCharScalar.class, IfExprCharScalarStringScalar.class, IfExprVarCharScalarStringScalar.class, + IfExprDecimalColumnColumn.class, IfExprDecimalColumnScalar.class, + IfExprDecimalScalarColumn.class, IfExprDecimalScalarScalar.class, + IfExprIntervalDayTimeColumnColumn.class, IfExprIntervalDayTimeColumnScalar.class, IfExprIntervalDayTimeScalarColumn.class, IfExprIntervalDayTimeScalarScalar.class, IfExprTimestampColumnColumn.class, IfExprTimestampColumnScalar.class, IfExprTimestampScalarColumn.class, IfExprTimestampScalarScalar.class, }) +@VectorizedExpressionsSupportDecimal64() public class GenericUDFIf extends GenericUDF { private transient ObjectInspector[] argumentOIs; private transient GenericUDFUtils.ReturnObjectInspectorResolver returnOIResolver; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java index 75bc478..6dcd3e4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java @@ -21,9 +21,6 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressionsSupportDecimal64; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColEqualLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColEqualLongScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongScalarEqualLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; @@ -39,6 +36,11 @@ LongScalarEqualLongColumn.class, LongScalarEqualDoubleColumn.class, DoubleScalarEqualLongColumn.class, DoubleScalarEqualDoubleColumn.class, + DecimalColEqualDecimalColumn.class, DecimalColEqualDecimalScalar.class, + DecimalScalarEqualDecimalColumn.class, + Decimal64ColEqualDecimal64Column.class, Decimal64ColEqualDecimal64Scalar.class, + Decimal64ScalarEqualDecimal64Column.class, + StringGroupColEqualStringGroupColumn.class, FilterStringGroupColEqualStringGroupColumn.class, StringGroupColEqualStringScalar.class, StringGroupColEqualVarCharScalar.class, StringGroupColEqualCharScalar.class, diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java index 90594b3..4383b3d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java @@ -21,9 +21,6 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressionsSupportDecimal64; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColGreaterEqualLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColGreaterEqualLongScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongScalarGreaterEqualLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; @@ -40,6 +37,11 @@ LongScalarGreaterEqualLongColumn.class, LongScalarGreaterEqualDoubleColumn.class, DoubleScalarGreaterEqualLongColumn.class, DoubleScalarGreaterEqualDoubleColumn.class, + DecimalColGreaterEqualDecimalColumn.class, DecimalColGreaterEqualDecimalScalar.class, + DecimalScalarGreaterEqualDecimalColumn.class, + Decimal64ColGreaterEqualDecimal64Column.class, Decimal64ColGreaterEqualDecimal64Scalar.class, + Decimal64ScalarGreaterEqualDecimal64Column.class, + StringGroupColGreaterEqualStringGroupColumn.class, FilterStringGroupColGreaterEqualStringGroupColumn.class, StringGroupColGreaterEqualStringScalar.class, StringGroupColGreaterEqualVarCharScalar.class, StringGroupColGreaterEqualCharScalar.class, diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java index 5176803..c24ae43 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java @@ -21,9 +21,6 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressionsSupportDecimal64; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColLessEqualLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColLessEqualLongScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongScalarLessEqualLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; @@ -40,6 +37,11 @@ LongScalarLessEqualLongColumn.class, LongScalarLessEqualDoubleColumn.class, DoubleScalarLessEqualLongColumn.class, DoubleScalarLessEqualDoubleColumn.class, + DecimalColLessEqualDecimalColumn.class, DecimalColLessEqualDecimalScalar.class, + DecimalScalarLessEqualDecimalColumn.class, + Decimal64ColLessEqualDecimal64Column.class, Decimal64ColLessEqualDecimal64Scalar.class, + Decimal64ScalarLessEqualDecimal64Column.class, + StringGroupColLessEqualStringGroupColumn.class, FilterStringGroupColLessEqualStringGroupColumn.class, StringGroupColLessEqualStringScalar.class, StringGroupColLessEqualVarCharScalar.class, StringGroupColLessEqualCharScalar.class, diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java index 938e64f..e86551d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java @@ -21,9 +21,6 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressionsSupportDecimal64; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColGreaterLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColGreaterLongScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongScalarGreaterLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; @@ -40,6 +37,11 @@ LongScalarGreaterLongColumn.class, LongScalarGreaterDoubleColumn.class, DoubleScalarGreaterLongColumn.class, DoubleScalarGreaterDoubleColumn.class, + DecimalColGreaterDecimalColumn.class, DecimalColGreaterDecimalScalar.class, + DecimalScalarGreaterDecimalColumn.class, + Decimal64ColGreaterDecimal64Column.class, Decimal64ColGreaterDecimal64Scalar.class, + Decimal64ScalarGreaterDecimal64Column.class, + StringGroupColGreaterStringGroupColumn.class, FilterStringGroupColGreaterStringGroupColumn.class, StringGroupColGreaterStringScalar.class, StringGroupColGreaterVarCharScalar.class, StringGroupColGreaterCharScalar.class, diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java index 734d394..709ac72 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java @@ -21,9 +21,6 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressionsSupportDecimal64; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColLessLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColLessLongScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongScalarLessLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; @@ -40,6 +37,11 @@ LongScalarLessLongColumn.class, LongScalarLessDoubleColumn.class, DoubleScalarLessLongColumn.class, DoubleScalarLessDoubleColumn.class, + DecimalColLessDecimalColumn.class, DecimalColLessDecimalScalar.class, + DecimalScalarLessDecimalColumn.class, + Decimal64ColLessDecimal64Column.class, Decimal64ColLessDecimal64Scalar.class, + Decimal64ScalarLessDecimal64Column.class, + StringGroupColLessStringGroupColumn.class, FilterStringGroupColLessStringGroupColumn.class, StringGroupColLessStringScalar.class, StringGroupColLessVarCharScalar.class, StringGroupColLessCharScalar.class, diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java index 00f8b7d..849b21d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java @@ -21,9 +21,6 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressionsSupportDecimal64; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColNotEqualLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColNotEqualLongScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongScalarNotEqualLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; @@ -39,6 +36,11 @@ LongScalarNotEqualLongColumn.class, LongScalarNotEqualDoubleColumn.class, DoubleScalarNotEqualLongColumn.class, DoubleScalarNotEqualDoubleColumn.class, + DecimalColNotEqualDecimalColumn.class, DecimalColNotEqualDecimalScalar.class, + DecimalScalarNotEqualDecimalColumn.class, + Decimal64ColNotEqualDecimal64Column.class, Decimal64ColNotEqualDecimal64Scalar.class, + Decimal64ScalarNotEqualDecimal64Column.class, + StringGroupColNotEqualStringGroupColumn.class, FilterStringGroupColNotEqualStringGroupColumn.class, StringGroupColNotEqualStringScalar.class, StringGroupColNotEqualVarCharScalar.class, StringGroupColNotEqualCharScalar.class, diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java index 4d0e85d..ee869db 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDecimalToTimestamp; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDoubleToTimestamp; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastLongToTimestamp; +import org.apache.hadoop.hive.ql.exec.vector.expressions.CastStringToTimestamp; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; @@ -49,7 +50,7 @@ @Description(name = "timestamp", value = "cast(date as timestamp) - Returns timestamp") @VectorizedExpressions({CastLongToTimestamp.class, CastDateToTimestamp.class, - CastDoubleToTimestamp.class, CastDecimalToTimestamp.class}) + CastDoubleToTimestamp.class, CastDecimalToTimestamp.class, CastStringToTimestamp.class}) public class GenericUDFTimestamp extends GenericUDF { private transient PrimitiveObjectInspector argumentOI; diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java index 70a481d..2039ee0 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java @@ -56,8 +56,8 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.IsNull; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColModuloLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColumnInList; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColEqualLongScalar; -import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColGreaterLongScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColEqualLongScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColGreaterLongScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.NotCol; import org.apache.hadoop.hive.ql.exec.vector.expressions.RoundWithNumDigitsDoubleToDouble; import org.apache.hadoop.hive.ql.exec.vector.expressions.SelectColumnIsFalse; diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmetic.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmetic.java index eb85823..a09daf3 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmetic.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmetic.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource.GenerationSpec; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -672,6 +673,16 @@ private void doVectorArithmeticTest(TypeInfo typeInfo1, VectorExpression vectorExpression = vectorizationContext.getVectorExpression(exprDesc); vectorExpression.transientInit(); + if (arithmeticTestMode == ArithmeticTestMode.VECTOR_EXPRESSION && + vectorExpression instanceof VectorUDFAdaptor) { + System.out.println( + "*NO NATIVE VECTOR EXPRESSION* typeInfo1 " + typeInfo1.toString() + + " typeInfo2 " + typeInfo2.toString() + + " arithmeticTestMode " + arithmeticTestMode + + " columnScalarMode " + columnScalarMode + + " vectorExpression " + vectorExpression.toString()); + } + String[] outputScratchTypeNames= vectorizationContext.getScratchColumnTypeNames(); DataTypePhysicalVariation[] outputDataTypePhysicalVariations = vectorizationContext.getScratchDataTypePhysicalVariations(); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorCastStatement.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorCastStatement.java index 2c82c4e..d4d8ef7 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorCastStatement.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorCastStatement.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource.GenerationSpec; import org.apache.hadoop.hive.ql.exec.vector.expressions.IdentityExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -157,13 +158,6 @@ public void testVarchar() throws Exception { } @Test - public void testBinary() throws Exception { - Random random = new Random(12882); - - doIfTests(random, "binary"); - } - - @Test public void testDecimal() throws Exception { Random random = new Random(9300); @@ -203,17 +197,15 @@ private void doIfTests(Random random, String typeName, continue; } - // BINARY conversions supported by GenericUDFDecimal, GenericUDFTimestamp. - if (primitiveCategory == PrimitiveCategory.BINARY) { - if (targetPrimitiveCategory == PrimitiveCategory.DECIMAL || - targetPrimitiveCategory == PrimitiveCategory.TIMESTAMP) { - continue; - } - } - - // DATE conversions supported by GenericUDFDecimal. + // DATE conversions NOT supported by integers, floating point, and GenericUDFDecimal. if (primitiveCategory == PrimitiveCategory.DATE) { - if (targetPrimitiveCategory == PrimitiveCategory.DECIMAL) { + if (targetPrimitiveCategory == PrimitiveCategory.BYTE || + targetPrimitiveCategory == PrimitiveCategory.SHORT || + targetPrimitiveCategory == PrimitiveCategory.INT || + targetPrimitiveCategory == PrimitiveCategory.LONG || + targetPrimitiveCategory == PrimitiveCategory.FLOAT || + targetPrimitiveCategory == PrimitiveCategory.DOUBLE || + targetPrimitiveCategory == PrimitiveCategory.DECIMAL) { continue; } } @@ -364,9 +356,10 @@ private void doIfTestOneCast(Random random, String typeName, " sourceTypeName " + typeName + " targetTypeName " + targetTypeName + " " + CastStmtTestMode.values()[v] + - " result is NULL " + (vectorResult == null ? "YES" : "NO") + + " result is NULL " + (vectorResult == null ? "YES" : "NO result " + vectorResult.toString()) + " does not match row-mode expected result is NULL " + - (expectedResult == null ? "YES" : "NO")); + (expectedResult == null ? "YES" : "NO result " + expectedResult.toString()) + + " row values " + Arrays.toString(randomRows[i])); } } else { @@ -387,7 +380,8 @@ private void doIfTestOneCast(Random random, String typeName, " result " + vectorResult.toString() + " (" + vectorResult.getClass().getSimpleName() + ")" + " does not match row-mode expected result " + expectedResult.toString() + - " (" + expectedResult.getClass().getSimpleName() + ")"); + " (" + expectedResult.getClass().getSimpleName() + ")" + + " row values " + Arrays.toString(randomRows[i])); } } } @@ -490,6 +484,14 @@ private boolean doVectorCastTest(TypeInfo typeInfo, TypeInfo targetTypeInfo, VectorExpression vectorExpression = vectorizationContext.getVectorExpression(exprDesc); vectorExpression.transientInit(); + if (castStmtTestMode == CastStmtTestMode.VECTOR_EXPRESSION && + vectorExpression instanceof VectorUDFAdaptor) { + System.out.println( + "*NO NATIVE VECTOR EXPRESSION* typeInfo " + typeInfo.toString() + + " castStmtTestMode " + castStmtTestMode + + " vectorExpression " + vectorExpression.toString()); + } + // System.out.println("*VECTOR EXPRESSION* " + vectorExpression.getClass().getSimpleName()); /* diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateAddSub.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateAddSub.java index da21aba..4dc01be 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateAddSub.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateAddSub.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource.GenerationSpec; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -448,6 +449,15 @@ private void doVectorDateAddSubTest(TypeInfo dateTimeStringTypeInfo, TypeInfo in VectorExpression vectorExpression = vectorizationContext.getVectorExpression(exprDesc); vectorExpression.transientInit(); + if (dateAddSubTestMode == DateAddSubTestMode.VECTOR_EXPRESSION && + vectorExpression instanceof VectorUDFAdaptor) { + System.out.println( + "*NO NATIVE VECTOR EXPRESSION* dateTimeStringTypeInfo " + dateTimeStringTypeInfo.toString() + + " dateAddSubTestMode " + dateAddSubTestMode + + " columnScalarMode " + columnScalarMode + + " vectorExpression " + vectorExpression.toString()); + } + VectorizedRowBatch batch = batchContext.createVectorizedRowBatch(); VectorExtractRow resultVectorExtractRow = new VectorExtractRow(); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateDiff.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateDiff.java index d5343ad..c5c5c72 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateDiff.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateDiff.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource.GenerationSpec; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -434,6 +435,16 @@ private void doVectorDateAddSubTest(TypeInfo dateTimeStringTypeInfo1, VectorExpression vectorExpression = vectorizationContext.getVectorExpression(exprDesc); vectorExpression.transientInit(); + if (dateDiffTestMode == DateDiffTestMode.VECTOR_EXPRESSION && + vectorExpression instanceof VectorUDFAdaptor) { + System.out.println( + "*NO NATIVE VECTOR EXPRESSION* dateTimeStringTypeInfo1 " + dateTimeStringTypeInfo1.toString() + + " dateTimeStringTypeInfo2 " + dateTimeStringTypeInfo2.toString() + + " dateDiffTestMode " + dateDiffTestMode + + " columnScalarMode " + columnScalarMode + + " vectorExpression " + vectorExpression.toString()); + } + VectorizedRowBatch batch = batchContext.createVectorizedRowBatch(); VectorExtractRow resultVectorExtractRow = new VectorExtractRow(); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterCompare.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterCompare.java new file mode 100644 index 0000000..8634fd3 --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterCompare.java @@ -0,0 +1,770 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import java.lang.reflect.Constructor; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation; +import org.apache.hadoop.hive.common.type.HiveChar; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; +import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory; +import org.apache.hadoop.hive.ql.exec.FunctionInfo; +import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorExtractRow; +import org.apache.hadoop.hive.ql.exec.vector.VectorRandomBatchSource; +import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource; +import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; +import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource.GenerationSpec; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFDateAdd; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFDateDiff; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFDateSub; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual; +import org.apache.hadoop.hive.serde2.io.HiveCharWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; +import org.apache.hadoop.hive.serde2.io.ShortWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.BooleanWritable; + +import junit.framework.Assert; + +import org.junit.Ignore; +import org.junit.Test; + +public class TestVectorFilterCompare { + + public TestVectorFilterCompare() { + // Arithmetic operations rely on getting conf from SessionState, need to initialize here. + SessionState ss = new SessionState(new HiveConf()); + ss.getConf().setVar(HiveConf.ConfVars.HIVE_COMPAT, "latest"); + SessionState.setCurrentSessionState(ss); + } + + @Test + public void testIntegers() throws Exception { + Random random = new Random(7743); + + doIntegerTests(random); + } + + @Test + public void testIntegerFloating() throws Exception { + Random random = new Random(7743); + + doIntegerFloatingTests(random); + } + + @Test + public void testFloating() throws Exception { + Random random = new Random(7743); + + doFloatingTests(random); + } + + @Test + public void testDecimal() throws Exception { + Random random = new Random(7743); + + doDecimalTests(random, /* tryDecimal64 */ false); + } + + @Test + public void testDecimal64() throws Exception { + Random random = new Random(7743); + + doDecimalTests(random, /* tryDecimal64 */ true); + } + + @Test + public void testTimestamp() throws Exception { + Random random = new Random(7743); + + doTests(random, TypeInfoFactory.timestampTypeInfo, TypeInfoFactory.timestampTypeInfo); + + doTests(random, TypeInfoFactory.timestampTypeInfo, TypeInfoFactory.longTypeInfo); + doTests(random, TypeInfoFactory.timestampTypeInfo, TypeInfoFactory.doubleTypeInfo); + + doTests(random, TypeInfoFactory.longTypeInfo, TypeInfoFactory.timestampTypeInfo); + doTests(random, TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.timestampTypeInfo); + } + + public enum FilterCompareTestMode { + ROW_MODE, + ADAPTOR, + FILTER_VECTOR_EXPRESSION, + COMPARE_VECTOR_EXPRESSION; + + static final int count = values().length; + } + + public enum ColumnScalarMode { + COLUMN_COLUMN, + COLUMN_SCALAR, + SCALAR_COLUMN; + + static final int count = values().length; + } + + private static TypeInfo[] integerTypeInfos = new TypeInfo[] { + TypeInfoFactory.byteTypeInfo, + TypeInfoFactory.shortTypeInfo, + TypeInfoFactory.intTypeInfo, + TypeInfoFactory.longTypeInfo + }; + + // We have test failures with FLOAT. Ignoring this issue for now. + private static TypeInfo[] floatingTypeInfos = new TypeInfo[] { + // TypeInfoFactory.floatTypeInfo, + TypeInfoFactory.doubleTypeInfo + }; + + private void doIntegerTests(Random random) + throws Exception { + for (TypeInfo typeInfo : integerTypeInfos) { + doTests(random, typeInfo, typeInfo); + } + } + + private void doIntegerFloatingTests(Random random) + throws Exception { + for (TypeInfo typeInfo1 : integerTypeInfos) { + for (TypeInfo typeInfo2 : floatingTypeInfos) { + doTests(random, typeInfo1, typeInfo2); + } + } + for (TypeInfo typeInfo1 : floatingTypeInfos) { + for (TypeInfo typeInfo2 : integerTypeInfos) { + doTests(random, typeInfo1, typeInfo2); + } + } + } + + private void doFloatingTests(Random random) + throws Exception { + for (TypeInfo typeInfo1 : floatingTypeInfos) { + for (TypeInfo typeInfo2 : floatingTypeInfos) { + doTests(random, typeInfo1, typeInfo2); + } + } + } + + private static TypeInfo[] decimalTypeInfos = new TypeInfo[] { + new DecimalTypeInfo(38, 18), + new DecimalTypeInfo(25, 2), + new DecimalTypeInfo(19, 4), + new DecimalTypeInfo(18, 10), + new DecimalTypeInfo(17, 3), + new DecimalTypeInfo(12, 2), + new DecimalTypeInfo(7, 1) + }; + + private void doDecimalTests(Random random, boolean tryDecimal64) + throws Exception { + for (TypeInfo typeInfo : decimalTypeInfos) { + doTests(random, typeInfo, typeInfo, tryDecimal64); + } + } + + private TypeInfo getOutputTypeInfo(GenericUDF genericUdfClone, + List objectInspectorList) + throws HiveException { + + ObjectInspector[] array = + objectInspectorList.toArray(new ObjectInspector[objectInspectorList.size()]); + ObjectInspector outputObjectInspector = genericUdfClone.initialize(array); + return TypeInfoUtils.getTypeInfoFromObjectInspector(outputObjectInspector); + } + + public enum Comparison { + EQUALS, + LESS_THAN, + LESS_THAN_EQUAL, + GREATER_THAN, + GREATER_THAN_EQUAL, + NOT_EQUALS; + } + + private TypeInfo getDecimalScalarTypeInfo(Object scalarObject) { + HiveDecimal dec = (HiveDecimal) scalarObject; + int precision = dec.precision(); + int scale = dec.scale(); + return new DecimalTypeInfo(precision, scale); + } + + private boolean checkDecimal64(boolean tryDecimal64, TypeInfo typeInfo) { + if (!tryDecimal64 || !(typeInfo instanceof DecimalTypeInfo)) { + return false; + } + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo; + boolean result = HiveDecimalWritable.isPrecisionDecimal64(decimalTypeInfo.getPrecision()); + return result; + } + + private void doTests(Random random, TypeInfo typeInfo1, TypeInfo typeInfo2, boolean tryDecimal64) + throws Exception { + for (ColumnScalarMode columnScalarMode : ColumnScalarMode.values()) { + doTestsWithDiffColumnScalar( + random, typeInfo1, typeInfo2, columnScalarMode, tryDecimal64); + } + } + + private void doTests(Random random, TypeInfo typeInfo1, TypeInfo typeInfo2) + throws Exception { + for (ColumnScalarMode columnScalarMode : ColumnScalarMode.values()) { + doTestsWithDiffColumnScalar( + random, typeInfo1, typeInfo2, columnScalarMode); + } + } + + private void doTestsWithDiffColumnScalar(Random random, TypeInfo typeInfo1, TypeInfo typeInfo2, + ColumnScalarMode columnScalarMode) + throws Exception { + doTestsWithDiffColumnScalar(random, typeInfo1, typeInfo2, columnScalarMode, false); + } + + private void doTestsWithDiffColumnScalar(Random random, TypeInfo typeInfo1, TypeInfo typeInfo2, + ColumnScalarMode columnScalarMode, boolean tryDecimal64) + throws Exception { + for (Comparison comparison : Comparison.values()) { + doTestsWithDiffColumnScalar( + random, typeInfo1, typeInfo2, columnScalarMode, comparison, tryDecimal64); + } + } + + private void doTestsWithDiffColumnScalar(Random random, TypeInfo typeInfo1, TypeInfo typeInfo2, + ColumnScalarMode columnScalarMode, Comparison comparison, boolean tryDecimal64) + throws Exception { + + String typeName1 = typeInfo1.getTypeName(); + PrimitiveCategory primitiveCategory1 = + ((PrimitiveTypeInfo) typeInfo1).getPrimitiveCategory(); + + String typeName2 = typeInfo2.getTypeName(); + PrimitiveCategory primitiveCategory2 = + ((PrimitiveTypeInfo) typeInfo2).getPrimitiveCategory(); + + List generationSpecList = new ArrayList(); + List explicitDataTypePhysicalVariationList = + new ArrayList(); + + List columns = new ArrayList(); + int columnNum = 0; + + ExprNodeDesc col1Expr; + Object scalar1Object = null; + final boolean decimal64Enable1 = checkDecimal64(tryDecimal64, typeInfo1); + if (columnScalarMode == ColumnScalarMode.COLUMN_COLUMN || + columnScalarMode == ColumnScalarMode.COLUMN_SCALAR) { + generationSpecList.add( + GenerationSpec.createSameType(typeInfo1)); + explicitDataTypePhysicalVariationList.add( + decimal64Enable1 ? + DataTypePhysicalVariation.DECIMAL_64 : + DataTypePhysicalVariation.NONE); + + String columnName = "col" + (columnNum++); + col1Expr = new ExprNodeColumnDesc(typeInfo1, columnName, "table", false); + columns.add(columnName); + } else { + scalar1Object = + VectorRandomRowSource.randomPrimitiveObject( + random, (PrimitiveTypeInfo) typeInfo1); + + // Adjust the decimal type to the scalar's type... + if (typeInfo1 instanceof DecimalTypeInfo) { + typeInfo1 = getDecimalScalarTypeInfo(scalar1Object); + } + + col1Expr = new ExprNodeConstantDesc(typeInfo1, scalar1Object); + } + ExprNodeDesc col2Expr; + Object scalar2Object = null; + final boolean decimal64Enable2 = checkDecimal64(tryDecimal64, typeInfo2); + if (columnScalarMode == ColumnScalarMode.COLUMN_COLUMN || + columnScalarMode == ColumnScalarMode.SCALAR_COLUMN) { + generationSpecList.add( + GenerationSpec.createSameType(typeInfo2)); + + explicitDataTypePhysicalVariationList.add( + decimal64Enable2 ? + DataTypePhysicalVariation.DECIMAL_64 : + DataTypePhysicalVariation.NONE); + + String columnName = "col" + (columnNum++); + col2Expr = new ExprNodeColumnDesc(typeInfo2, columnName, "table", false); + columns.add(columnName); + } else { + scalar2Object = + VectorRandomRowSource.randomPrimitiveObject( + random, (PrimitiveTypeInfo) typeInfo2); + + // Adjust the decimal type to the scalar's type... + if (typeInfo2 instanceof DecimalTypeInfo) { + typeInfo2 = getDecimalScalarTypeInfo(scalar2Object); + } + + col2Expr = new ExprNodeConstantDesc(typeInfo2, scalar2Object); + } + + List objectInspectorList = new ArrayList(); + objectInspectorList.add(VectorRandomRowSource.getObjectInspector(typeInfo1)); + objectInspectorList.add(VectorRandomRowSource.getObjectInspector(typeInfo2)); + + List children = new ArrayList(); + children.add(col1Expr); + children.add(col2Expr); + + //---------------------------------------------------------------------------------------------- + + String[] columnNames = columns.toArray(new String[0]); + + VectorRandomRowSource rowSource = new VectorRandomRowSource(); + + rowSource.initGenerationSpecSchema( + random, generationSpecList, /* maxComplexDepth */ 0, /* allowNull */ true, + explicitDataTypePhysicalVariationList); + + Object[][] randomRows = rowSource.randomRows(100000); + + VectorRandomBatchSource batchSource = + VectorRandomBatchSource.createInterestingBatches( + random, + rowSource, + randomRows, + null); + + GenericUDF genericUdf; + switch (comparison) { + case EQUALS: + genericUdf = new GenericUDFOPEqual(); + break; + case LESS_THAN: + genericUdf = new GenericUDFOPLessThan(); + break; + case LESS_THAN_EQUAL: + genericUdf = new GenericUDFOPEqualOrLessThan(); + break; + case GREATER_THAN: + genericUdf = new GenericUDFOPGreaterThan(); + break; + case GREATER_THAN_EQUAL: + genericUdf = new GenericUDFOPEqualOrGreaterThan(); + break; + case NOT_EQUALS: + genericUdf = new GenericUDFOPNotEqual(); + break; + default: + throw new RuntimeException("Unexpected arithmetic " + comparison); + } + + ObjectInspector[] objectInspectors = + objectInspectorList.toArray(new ObjectInspector[objectInspectorList.size()]); + ObjectInspector outputObjectInspector = null; + try { + outputObjectInspector = genericUdf.initialize(objectInspectors); + } catch (Exception e) { + Assert.fail(e.toString()); + } + + TypeInfo outputTypeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(outputObjectInspector); + + ExprNodeGenericFuncDesc exprDesc = + new ExprNodeGenericFuncDesc(outputTypeInfo, genericUdf, children); + + final int rowCount = randomRows.length; + Object[][] resultObjectsArray = new Object[FilterCompareTestMode.count][]; + for (int i = 0; i < FilterCompareTestMode.count; i++) { + + Object[] resultObjects = new Object[rowCount]; + resultObjectsArray[i] = resultObjects; + + FilterCompareTestMode filterCompareTestMode = FilterCompareTestMode.values()[i]; + switch (filterCompareTestMode) { + case ROW_MODE: + doRowFilterCompareTest( + typeInfo1, + typeInfo2, + columns, + children, + exprDesc, + comparison, + randomRows, + columnScalarMode, + rowSource.rowStructObjectInspector(), + outputTypeInfo, + resultObjects); + break; + case ADAPTOR: + case FILTER_VECTOR_EXPRESSION: + case COMPARE_VECTOR_EXPRESSION: + doVectorFilterCompareTest( + typeInfo1, + typeInfo2, + columns, + columnNames, + rowSource.typeInfos(), + rowSource.dataTypePhysicalVariations(), + children, + exprDesc, + comparison, + filterCompareTestMode, + columnScalarMode, + batchSource, + exprDesc.getWritableObjectInspector(), + outputTypeInfo, + resultObjects); + break; + default: + throw new RuntimeException("Unexpected IF statement test mode " + filterCompareTestMode); + } + } + + for (int i = 0; i < rowCount; i++) { + // Row-mode is the expected value. + Object expectedResult = resultObjectsArray[0][i]; + + for (int v = 1; v < FilterCompareTestMode.count; v++) { + FilterCompareTestMode filterCompareTestMode = FilterCompareTestMode.values()[v]; + Object vectorResult = resultObjectsArray[v][i]; + if (filterCompareTestMode == FilterCompareTestMode.FILTER_VECTOR_EXPRESSION && + expectedResult == null && + vectorResult != null) { + // This is OK. + boolean vectorBoolean = ((BooleanWritable) vectorResult).get(); + if (vectorBoolean) { + Assert.fail( + "Row " + i + + " typeName1 " + typeName1 + + " typeName2 " + typeName2 + + " outputTypeName " + outputTypeInfo.getTypeName() + + " " + comparison + + " " + filterCompareTestMode + + " " + columnScalarMode + + " result is NOT NULL and true" + + " does not match row-mode expected result is NULL which means false here" + + (columnScalarMode == ColumnScalarMode.SCALAR_COLUMN ? + " scalar1 " + scalar1Object.toString() : "") + + " row values " + Arrays.toString(randomRows[i]) + + (columnScalarMode == ColumnScalarMode.COLUMN_SCALAR ? + " scalar2 " + scalar2Object.toString() : "")); + } + } else if (expectedResult == null || vectorResult == null) { + if (expectedResult != null || vectorResult != null) { + Assert.fail( + "Row " + i + + " typeName1 " + typeName1 + + " typeName2 " + typeName2 + + " outputTypeName " + outputTypeInfo.getTypeName() + + " " + comparison + + " " + filterCompareTestMode + + " " + columnScalarMode + + " result is NULL " + (vectorResult == null) + + " does not match row-mode expected result is NULL " + (expectedResult == null) + + (columnScalarMode == ColumnScalarMode.SCALAR_COLUMN ? + " scalar1 " + scalar1Object.toString() : "") + + " row values " + Arrays.toString(randomRows[i]) + + (columnScalarMode == ColumnScalarMode.COLUMN_SCALAR ? + " scalar2 " + scalar2Object.toString() : "")); + } + } else { + + if (!expectedResult.equals(vectorResult)) { + Assert.fail( + "Row " + i + + " typeName1 " + typeName1 + + " typeName2 " + typeName2 + + " outputTypeName " + outputTypeInfo.getTypeName() + + " " + comparison + + " " + filterCompareTestMode + + " " + columnScalarMode + + " result " + vectorResult.toString() + + " (" + vectorResult.getClass().getSimpleName() + ")" + + " does not match row-mode expected result " + expectedResult.toString() + + " (" + expectedResult.getClass().getSimpleName() + ")" + + (columnScalarMode == ColumnScalarMode.SCALAR_COLUMN ? + " scalar1 " + scalar1Object.toString() : "") + + " row values " + Arrays.toString(randomRows[i]) + + (columnScalarMode == ColumnScalarMode.COLUMN_SCALAR ? + " scalar2 " + scalar2Object.toString() : "")); + } + } + } + } + } + + private void doRowFilterCompareTest(TypeInfo typeInfo1, + TypeInfo typeInfo2, + List columns, List children, + ExprNodeGenericFuncDesc exprDesc, + Comparison comparison, + Object[][] randomRows, ColumnScalarMode columnScalarMode, + ObjectInspector rowInspector, + TypeInfo outputTypeInfo, Object[] resultObjects) throws Exception { + + /* + System.out.println( + "*DEBUG* typeInfo " + typeInfo1.toString() + + " typeInfo2 " + typeInfo2 + + " filterCompareTestMode ROW_MODE" + + " columnScalarMode " + columnScalarMode + + " exprDesc " + exprDesc.toString()); + */ + + HiveConf hiveConf = new HiveConf(); + ExprNodeEvaluator evaluator = + ExprNodeEvaluatorFactory.get(exprDesc, hiveConf); + evaluator.initialize(rowInspector); + + ObjectInspector objectInspector = + TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo( + outputTypeInfo); + + final int rowCount = randomRows.length; + for (int i = 0; i < rowCount; i++) { + Object[] row = randomRows[i]; + Object result = evaluator.evaluate(row); + Object copyResult = null; + try { + copyResult = + ObjectInspectorUtils.copyToStandardObject( + result, objectInspector, ObjectInspectorCopyOption.WRITABLE); + } catch (Exception e) { + Assert.fail(e.toString()); + } + resultObjects[i] = copyResult; + } + } + + private void extractResultObjects(VectorizedRowBatch batch, int rowIndex, + VectorExtractRow resultVectorExtractRow, Object[] scrqtchRow, + ObjectInspector objectInspector, Object[] resultObjects) { + + boolean selectedInUse = batch.selectedInUse; + int[] selected = batch.selected; + for (int logicalIndex = 0; logicalIndex < batch.size; logicalIndex++) { + final int batchIndex = (selectedInUse ? selected[logicalIndex] : logicalIndex); + resultVectorExtractRow.extractRow(batch, batchIndex, scrqtchRow); + + Object copyResult = + ObjectInspectorUtils.copyToStandardObject( + scrqtchRow[0], objectInspector, ObjectInspectorCopyOption.WRITABLE); + resultObjects[rowIndex++] = copyResult; + } + } + + private void doVectorFilterCompareTest(TypeInfo typeInfo1, + TypeInfo typeInfo2, + List columns, + String[] columnNames, + TypeInfo[] typeInfos, DataTypePhysicalVariation[] dataTypePhysicalVariations, + List children, + ExprNodeGenericFuncDesc exprDesc, + Comparison comparison, + FilterCompareTestMode filterCompareTestMode, ColumnScalarMode columnScalarMode, + VectorRandomBatchSource batchSource, + ObjectInspector objectInspector, + TypeInfo outputTypeInfo, Object[] resultObjects) + throws Exception { + + HiveConf hiveConf = new HiveConf(); + if (filterCompareTestMode == FilterCompareTestMode.ADAPTOR) { + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_TEST_VECTOR_ADAPTOR_OVERRIDE, true); + + // Don't use DECIMAL_64 with the VectorUDFAdaptor. + dataTypePhysicalVariations = null; + } + + VectorizationContext vectorizationContext = + new VectorizationContext( + "name", + columns, + Arrays.asList(typeInfos), + dataTypePhysicalVariations == null ? null : Arrays.asList(dataTypePhysicalVariations), + hiveConf); + final VectorExpressionDescriptor.Mode mode; + switch (filterCompareTestMode) { + case ADAPTOR: + case COMPARE_VECTOR_EXPRESSION: + mode = VectorExpressionDescriptor.Mode.PROJECTION; + break; + case FILTER_VECTOR_EXPRESSION: + mode = VectorExpressionDescriptor.Mode.FILTER; + break; + default: + throw new RuntimeException("Unexpected filter compare mode " + filterCompareTestMode); + } + VectorExpression vectorExpression = + vectorizationContext.getVectorExpression( + exprDesc, mode); + vectorExpression.transientInit(); + + if (filterCompareTestMode == FilterCompareTestMode.COMPARE_VECTOR_EXPRESSION && + vectorExpression instanceof VectorUDFAdaptor) { + System.out.println( + "*NO NATIVE VECTOR EXPRESSION* typeInfo1 " + typeInfo1.toString() + + " typeInfo2 " + typeInfo2.toString() + + " " + comparison + " " + + " filterCompareTestMode " + filterCompareTestMode + + " columnScalarMode " + columnScalarMode + + " vectorExpression " + vectorExpression.toString()); + } + + String[] outputScratchTypeNames= vectorizationContext.getScratchColumnTypeNames(); + DataTypePhysicalVariation[] outputDataTypePhysicalVariations = + vectorizationContext.getScratchDataTypePhysicalVariations(); + + VectorizedRowBatchCtx batchContext = + new VectorizedRowBatchCtx( + columnNames, + typeInfos, + dataTypePhysicalVariations, + /* dataColumnNums */ null, + /* partitionColumnCount */ 0, + /* virtualColumnCount */ 0, + /* neededVirtualColumns */ null, + outputScratchTypeNames, + outputDataTypePhysicalVariations); + + VectorizedRowBatch batch = batchContext.createVectorizedRowBatch(); + + VectorExtractRow resultVectorExtractRow = new VectorExtractRow(); + final int outputColumnNum = vectorExpression.getOutputColumnNum(); + resultVectorExtractRow.init( + new TypeInfo[] { outputTypeInfo }, new int[] { outputColumnNum }); + Object[] scrqtchRow = new Object[1]; + + // System.out.println("*VECTOR EXPRESSION* " + vectorExpression.getClass().getSimpleName()); + + /* + System.out.println( + "*DEBUG* typeInfo1 " + typeInfo1.toString() + + " typeInfo2 " + typeInfo2.toString() + + " " + comparison + " " + + " filterCompareTestMode " + filterCompareTestMode + + " columnScalarMode " + columnScalarMode + + " vectorExpression " + vectorExpression.toString()); + */ + + final boolean isFilter = (mode == VectorExpressionDescriptor.Mode.FILTER); + boolean copySelectedInUse = false; + int[] copySelected = new int[VectorizedRowBatch.DEFAULT_SIZE]; + + batchSource.resetBatchIteration(); + int rowIndex = 0; + while (true) { + if (!batchSource.fillNextBatch(batch)) { + break; + } + final int originalBatchSize = batch.size; + if (isFilter) { + copySelectedInUse = batch.selectedInUse; + if (batch.selectedInUse) { + System.arraycopy(batch.selected, 0, copySelected, 0, originalBatchSize); + } + } + + // In filter mode, the batch size can be made smaller. + vectorExpression.evaluate(batch); + + if (!isFilter) { + extractResultObjects(batch, rowIndex, resultVectorExtractRow, scrqtchRow, + objectInspector, resultObjects); + } else { + final int currentBatchSize = batch.size; + if (copySelectedInUse && batch.selectedInUse) { + int selectIndex = 0; + for (int i = 0; i < originalBatchSize; i++) { + final int originalBatchIndex = copySelected[i]; + final boolean booleanResult; + if (selectIndex < currentBatchSize && batch.selected[selectIndex] == originalBatchIndex) { + booleanResult = true; + selectIndex++; + } else { + booleanResult = false; + } + resultObjects[rowIndex + i] = new BooleanWritable(booleanResult); + } + } else if (batch.selectedInUse) { + int selectIndex = 0; + for (int i = 0; i < originalBatchSize; i++) { + final boolean booleanResult; + if (selectIndex < currentBatchSize && batch.selected[selectIndex] == i) { + booleanResult = true; + selectIndex++; + } else { + booleanResult = false; + } + resultObjects[rowIndex + i] = new BooleanWritable(booleanResult); + } + } else if (currentBatchSize == 0) { + // Whole batch got zapped. + for (int i = 0; i < originalBatchSize; i++) { + resultObjects[rowIndex + i] = new BooleanWritable(false); + } + } else { + // Every row kept. + for (int i = 0; i < originalBatchSize; i++) { + resultObjects[rowIndex + i] = new BooleanWritable(true); + } + } + } + rowIndex += originalBatchSize; + } + } +} diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorIfStatement.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorIfStatement.java index 9d78ca6..58e32ca 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorIfStatement.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorIfStatement.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -155,6 +156,20 @@ public void testDecimalSmall() throws Exception { doIfTests(random, "decimal(10,4)"); } + @Test + public void testDecimal64() throws Exception { + Random random = new Random(12882); + + doIfTestsWithDiffColumnScalar( + random, "decimal(10,4)", ColumnScalarMode.COLUMN_COLUMN, DataTypePhysicalVariation.DECIMAL_64, false, false); + doIfTestsWithDiffColumnScalar( + random, "decimal(10,4)", ColumnScalarMode.COLUMN_SCALAR, DataTypePhysicalVariation.DECIMAL_64, false, false); + doIfTestsWithDiffColumnScalar( + random, "decimal(10,4)", ColumnScalarMode.SCALAR_COLUMN, DataTypePhysicalVariation.DECIMAL_64, false, false); + doIfTestsWithDiffColumnScalar( + random, "decimal(10,4)", ColumnScalarMode.SCALAR_SCALAR, DataTypePhysicalVariation.DECIMAL_64, false, false); + } + public enum IfStmtTestMode { ROW_MODE, ADAPTOR_WHEN, @@ -280,22 +295,6 @@ private void doIfTestsWithDiffColumnScalar(Random random, String typeName, String[] columnNames = columns.toArray(new String[0]); - String[] outputScratchTypeNames = new String[] { typeName }; - DataTypePhysicalVariation[] outputDataTypePhysicalVariations = - new DataTypePhysicalVariation[] { dataTypePhysicalVariation }; - - VectorizedRowBatchCtx batchContext = - new VectorizedRowBatchCtx( - columnNames, - rowSource.typeInfos(), - rowSource.dataTypePhysicalVariations(), - /* dataColumnNums */ null, - /* partitionColumnCount */ 0, - /* virtualColumnCount */ 0, - /* neededVirtualColumns */ null, - outputScratchTypeNames, - outputDataTypePhysicalVariations); - Object[][] randomRows = rowSource.randomRows(100000); VectorRandomBatchSource batchSource = @@ -324,13 +323,13 @@ private void doIfTestsWithDiffColumnScalar(Random random, String typeName, doVectorIfTest( typeInfo, columns, + columnNames, rowSource.typeInfos(), rowSource.dataTypePhysicalVariations(), children, ifStmtTestMode, columnScalarMode, batchSource, - batchContext, resultObjects); break; default: @@ -354,14 +353,6 @@ private void doIfTestsWithDiffColumnScalar(Random random, String typeName, } } else { - if (isDecimal64 && expectedResult instanceof LongWritable) { - - HiveDecimalWritable expectedHiveDecimalWritable = new HiveDecimalWritable(0); - expectedHiveDecimalWritable.deserialize64( - ((LongWritable) expectedResult).get(), decimal64Scale); - expectedResult = expectedHiveDecimalWritable; - } - if (!expectedResult.equals(vectorResult)) { Assert.fail( "Row " + i + " " + IfStmtTestMode.values()[v] + @@ -418,10 +409,11 @@ private void extractResultObjects(VectorizedRowBatch batch, int rowIndex, private void doVectorIfTest(TypeInfo typeInfo, List columns, + String[] columnNames, TypeInfo[] typeInfos, DataTypePhysicalVariation[] dataTypePhysicalVariations, List children, IfStmtTestMode ifStmtTestMode, ColumnScalarMode columnScalarMode, - VectorRandomBatchSource batchSource, VectorizedRowBatchCtx batchContext, + VectorRandomBatchSource batchSource, Object[] resultObjects) throws Exception { @@ -453,10 +445,36 @@ private void doVectorIfTest(TypeInfo typeInfo, hiveConf); VectorExpression vectorExpression = vectorizationContext.getVectorExpression(exprDesc); + if (ifStmtTestMode == IfStmtTestMode.VECTOR_EXPRESSION && + vectorExpression instanceof VectorUDFAdaptor) { + System.out.println( + "*NO NATIVE VECTOR EXPRESSION* typeInfo " + typeInfo.toString() + + " ifStmtTestMode " + ifStmtTestMode + + " columnScalarMode " + columnScalarMode + + " vectorExpression " + vectorExpression.toString()); + } + + String[] outputScratchTypeNames= vectorizationContext.getScratchColumnTypeNames(); + DataTypePhysicalVariation[] outputDataTypePhysicalVariations = + vectorizationContext.getScratchDataTypePhysicalVariations(); + + VectorizedRowBatchCtx batchContext = + new VectorizedRowBatchCtx( + columnNames, + typeInfos, + dataTypePhysicalVariations, + /* dataColumnNums */ null, + /* partitionColumnCount */ 0, + /* virtualColumnCount */ 0, + /* neededVirtualColumns */ null, + outputScratchTypeNames, + outputDataTypePhysicalVariations); + VectorizedRowBatch batch = batchContext.createVectorizedRowBatch(); VectorExtractRow resultVectorExtractRow = new VectorExtractRow(); - resultVectorExtractRow.init(new TypeInfo[] { typeInfo }, new int[] { columns.size() }); + resultVectorExtractRow.init( + new TypeInfo[] { typeInfo }, new int[] { vectorExpression.getOutputColumnNum() }); Object[] scrqtchRow = new Object[1]; // System.out.println("*VECTOR EXPRESSION* " + vectorExpression.getClass().getSimpleName()); @@ -466,7 +484,7 @@ private void doVectorIfTest(TypeInfo typeInfo, "*DEBUG* typeInfo " + typeInfo.toString() + " ifStmtTestMode " + ifStmtTestMode + " columnScalarMode " + columnScalarMode + - " vectorExpression " + vectorExpression.getClass().getSimpleName()); + " vectorExpression " + vectorExpression.toString()); */ batchSource.resetBatchIteration(); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorNegative.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorNegative.java index 289efb9..54c085b 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorNegative.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorNegative.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource.GenerationSpec; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -406,6 +407,14 @@ private void doVectorArithmeticTest(TypeInfo typeInfo, VectorExpression vectorExpression = vectorizationContext.getVectorExpression(exprDesc); vectorExpression.transientInit(); + if (negativeTestMode == NegativeTestMode.VECTOR_EXPRESSION && + vectorExpression instanceof VectorUDFAdaptor) { + System.out.println( + "*NO NATIVE VECTOR EXPRESSION* typeInfo " + typeInfo.toString() + + " negativeTestMode " + negativeTestMode + + " vectorExpression " + vectorExpression.toString()); + } + String[] outputScratchTypeNames= vectorizationContext.getScratchColumnTypeNames(); VectorizedRowBatchCtx batchContext = diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringConcat.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringConcat.java index 3d030e6..69fd70c 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringConcat.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringConcat.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource.GenerationSpec; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; @@ -400,6 +401,16 @@ private void doVectorStringConcatTest(TypeInfo stringTypeInfo1, TypeInfo stringT VectorExpression vectorExpression = vectorizationContext.getVectorExpression(exprDesc); vectorExpression.transientInit(); + if (stringConcatTestMode == StringConcatTestMode.VECTOR_EXPRESSION && + vectorExpression instanceof VectorUDFAdaptor) { + System.out.println( + "*NO NATIVE VECTOR EXPRESSION* stringTypeInfo1 " + stringTypeInfo1.toString() + + " stringTypeInfo2 " + stringTypeInfo2.toString() + + " stringConcatTestMode " + stringConcatTestMode + + " columnScalarMode " + columnScalarMode + + " vectorExpression " + vectorExpression.toString()); + } + VectorizedRowBatch batch = batchContext.createVectorizedRowBatch(); VectorExtractRow resultVectorExtractRow = new VectorExtractRow(); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringUnary.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringUnary.java index 4398554..f029358 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringUnary.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringUnary.java @@ -38,7 +38,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource.GenerationSpec; import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource.StringGenerationOption; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.TestVectorTimestampExtract.TimestampExtractTestMode; +import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -104,8 +104,6 @@ private void doTests(Random random, String typeName) doTests(random, typeName, "trim"); doTests(random, typeName, "initcap"); - - doTests(random, typeName, "hex"); } doTests(random, typeName, "lower"); @@ -341,6 +339,14 @@ private void doVectorIfTest(TypeInfo typeInfo, TypeInfo targetTypeInfo, hiveConf); VectorExpression vectorExpression = vectorizationContext.getVectorExpression(exprDesc); + if (stringUnaryTestMode == StringUnaryTestMode.VECTOR_EXPRESSION && + vectorExpression instanceof VectorUDFAdaptor) { + System.out.println( + "*NO NATIVE VECTOR EXPRESSION* typeInfo " + typeInfo.toString() + + " stringUnaryTestMode " + stringUnaryTestMode + + " vectorExpression " + vectorExpression.toString()); + } + VectorizedRowBatch batch = batchContext.createVectorizedRowBatch(); VectorExtractRow resultVectorExtractRow = new VectorExtractRow(); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorSubStr.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorSubStr.java index 62d296d..694f6f7 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorSubStr.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorSubStr.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource.GenerationSpec; import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource.StringGenerationOption; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -320,6 +321,14 @@ private void doVectorIfTest(TypeInfo typeInfo, TypeInfo targetTypeInfo, hiveConf); VectorExpression vectorExpression = vectorizationContext.getVectorExpression(exprDesc); + if (subStrTestMode == SubStrTestMode.VECTOR_EXPRESSION && + vectorExpression instanceof VectorUDFAdaptor) { + System.out.println( + "*NO NATIVE VECTOR EXPRESSION* typeInfo " + typeInfo.toString() + + " subStrTestMode " + subStrTestMode + + " vectorExpression " + vectorExpression.toString()); + } + VectorizedRowBatch batch = batchContext.createVectorizedRowBatch(); VectorExtractRow resultVectorExtractRow = new VectorExtractRow(); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExtract.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExtract.java index 3220f4c..5d5e4c9 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExtract.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExtract.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorRandomBatchSource; import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource; import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource.GenerationSpec; +import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; @@ -395,6 +396,14 @@ private boolean doVectorCastTest(TypeInfo dateTimeStringTypeInfo, VectorExpression vectorExpression = vectorizationContext.getVectorExpression(exprDesc); vectorExpression.transientInit(); + if (timestampExtractTestMode == TimestampExtractTestMode.VECTOR_EXPRESSION && + vectorExpression instanceof VectorUDFAdaptor) { + System.out.println( + "*NO NATIVE VECTOR EXPRESSION* dateTimeStringTypeInfo " + dateTimeStringTypeInfo.toString() + + " timestampExtractTestMode " + timestampExtractTestMode + + " vectorExpression " + vectorExpression.toString()); + } + // System.out.println("*VECTOR EXPRESSION* " + vectorExpression.getClass().getSimpleName()); /* diff --git ql/src/test/results/clientpositive/convert_decimal64_to_decimal.q.out ql/src/test/results/clientpositive/convert_decimal64_to_decimal.q.out index 8e538d2..5189957 100644 --- ql/src/test/results/clientpositive/convert_decimal64_to_decimal.q.out +++ ql/src/test/results/clientpositive/convert_decimal64_to_decimal.q.out @@ -181,7 +181,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 33:decimal(4,2))(children: ConvertDecimal64ToDecimal(col 19:decimal(4,2)/DECIMAL_64) -> 33:decimal(4,2)) + predicateExpression: SelectColumnIsNotNull(col 19:decimal(4,2)/DECIMAL_64) predicate: decimal0402_col_20 is not null (type: boolean) Statistics: Num rows: 1000 Data size: 2087309 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -199,7 +199,7 @@ STAGE PLANS: 0 _col0 (type: decimal(9,2)) 1 _col1 (type: decimal(9,2)) Map Join Vectorization: - bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 19:decimal(9,2)/DECIMAL_64) -> 34:decimal(9,2) + bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 19:decimal(9,2)/DECIMAL_64) -> 33:decimal(9,2) bigTableValueExpressions: col 14:smallint className: VectorMapJoinOperator native: false @@ -244,7 +244,7 @@ STAGE PLANS: includeColumns: [14, 19] dataColumns: tinyint_col_1:tinyint, float_col_2:float, bigint_col_3:bigint, boolean_col_4:boolean, decimal0202_col_5:decimal(2,2)/DECIMAL_64, decimal1612_col_6:decimal(16,12)/DECIMAL_64, double_col_7:double, char0205_col_8:char(205), bigint_col_9:bigint, decimal1202_col_10:decimal(12,2)/DECIMAL_64, boolean_col_11:boolean, double_col_12:double, decimal2208_col_13:decimal(22,8), decimal3722_col_14:decimal(37,22), smallint_col_15:smallint, decimal2824_col_16:decimal(28,24), boolean_col_17:boolean, float_col_18:float, timestamp_col_19:timestamp, decimal0402_col_20:decimal(4,2)/DECIMAL_64, char0208_col_21:char(208), char0077_col_22:char(77), decimal2915_col_23:decimal(29,15), char0234_col_24:char(234), timestamp_col_25:timestamp, tinyint_col_26:tinyint, decimal3635_col_27:decimal(36,35), boolean_col_28:boolean, float_col_29:float, smallint_col_30:smallint, varchar0200_col_31:varchar(200), boolean_col_32:boolean partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(4,2), decimal(9,2)] + scratchColumnTypeNames: [decimal(9,2)] Local Work: Map Reduce Local Work Reduce Vectorization: diff --git ql/src/test/results/clientpositive/llap/convert_decimal64_to_decimal.q.out ql/src/test/results/clientpositive/llap/convert_decimal64_to_decimal.q.out index cbc6b25..fddd2cb 100644 --- ql/src/test/results/clientpositive/llap/convert_decimal64_to_decimal.q.out +++ ql/src/test/results/clientpositive/llap/convert_decimal64_to_decimal.q.out @@ -163,7 +163,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 20:decimal(8,1))(children: ConvertDecimal64ToDecimal(col 3:decimal(8,1)/DECIMAL_64) -> 20:decimal(8,1)) + predicateExpression: SelectColumnIsNotNull(col 3:decimal(8,1)/DECIMAL_64) predicate: decimal0801_col_4 is not null (type: boolean) Statistics: Num rows: 1026 Data size: 109272 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -181,7 +181,7 @@ STAGE PLANS: 0 _col0 (type: decimal(9,2)) 1 _col1 (type: decimal(9,2)) Map Join Vectorization: - bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 3:decimal(9,2)/DECIMAL_64) -> 21:decimal(9,2) + bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 3:decimal(9,2)/DECIMAL_64) -> 20:decimal(9,2) className: VectorMapJoinOperator native: false nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true @@ -229,7 +229,7 @@ STAGE PLANS: includeColumns: [3] dataColumns: float_col_1:float, varchar0037_col_2:varchar(37), decimal2912_col_3:decimal(29,12), decimal0801_col_4:decimal(8,1)/DECIMAL_64, timestamp_col_5:timestamp, boolean_col_6:boolean, string_col_7:string, tinyint_col_8:tinyint, boolean_col_9:boolean, decimal1614_col_10:decimal(16,14)/DECIMAL_64, boolean_col_11:boolean, float_col_12:float, char0116_col_13:char(116), boolean_col_14:boolean, string_col_15:string, double_col_16:double, string_col_17:string, bigint_col_18:bigint, int_col_19:int partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(8,1), decimal(9,2), bigint] + scratchColumnTypeNames: [decimal(9,2), bigint] Map 3 Map Operator Tree: TableScan @@ -243,7 +243,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 33:decimal(4,2))(children: ConvertDecimal64ToDecimal(col 19:decimal(4,2)/DECIMAL_64) -> 33:decimal(4,2)) + predicateExpression: SelectColumnIsNotNull(col 19:decimal(4,2)/DECIMAL_64) predicate: decimal0402_col_20 is not null (type: boolean) Statistics: Num rows: 950 Data size: 104800 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -282,7 +282,7 @@ STAGE PLANS: includeColumns: [14, 19] dataColumns: tinyint_col_1:tinyint, float_col_2:float, bigint_col_3:bigint, boolean_col_4:boolean, decimal0202_col_5:decimal(2,2)/DECIMAL_64, decimal1612_col_6:decimal(16,12)/DECIMAL_64, double_col_7:double, char0205_col_8:char(205), bigint_col_9:bigint, decimal1202_col_10:decimal(12,2)/DECIMAL_64, boolean_col_11:boolean, double_col_12:double, decimal2208_col_13:decimal(22,8), decimal3722_col_14:decimal(37,22), smallint_col_15:smallint, decimal2824_col_16:decimal(28,24), boolean_col_17:boolean, float_col_18:float, timestamp_col_19:timestamp, decimal0402_col_20:decimal(4,2)/DECIMAL_64, char0208_col_21:char(208), char0077_col_22:char(77), decimal2915_col_23:decimal(29,15), char0234_col_24:char(234), timestamp_col_25:timestamp, tinyint_col_26:tinyint, decimal3635_col_27:decimal(36,35), boolean_col_28:boolean, float_col_29:float, smallint_col_30:smallint, varchar0200_col_31:varchar(200), boolean_col_32:boolean partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(4,2)] + scratchColumnTypeNames: [] Reducer 2 Execution mode: vectorized, llap Reduce Vectorization: diff --git ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out index ab083e7..d5f79f7 100644 --- ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out +++ ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out @@ -523,8 +523,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [4, 22, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 35, 38, 40, 43, 44] - selectExpressions: IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 21:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 22:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 21:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprStringScalarStringScalar(col 20:boolean, val Many, val Huge number)(children: LongColLessLongScalar(col 4:int, val 100) -> 20:boolean) -> 21:string) -> 22:string) -> 21:string) -> 22:string, IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 23:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 24:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 23:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprColumnNull(col 20:boolean, col 21:string, null)(children: LongColLessLongScalar(col 4:int, val 100) -> 20:boolean, ConstantVectorExpression(val Many) -> 21:string) -> 23:string) -> 24:string) -> 23:string) -> 24:string, IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 23:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 25:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 23:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprNullNull(null, null) -> 23:string) -> 25:string) -> 23:string) -> 25:string, IfExprLongColumnLongColumn(col 17:boolean, col 18:date, col 19:date)(children: StringGroupColEqualCharScalar(col 14:char(10), val SHIP) -> 17:boolean, VectorUDFDateAddColScalar(col 10:date, val 10) -> 18:date, VectorUDFDateAddColScalar(col 10:date, val 5) -> 19:date) -> 26:date, IfExprDoubleColumnLongScalar(col 17:boolean, col 28:double, val 0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 27:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 27:double) -> 28:double) -> 27:double, IfExprDoubleColumnDoubleScalar(col 17:boolean, col 29:double, val 0.0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 28:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 28:double) -> 29:double) -> 28:double, IfExprNullColumn(col 17:boolean, null, col 46)(children: StringGroupColEqualStringScalar(col 23:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 17:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 46:decimal(10,2)) -> 30:decimal(10,2), IfExprColumnNull(col 18:boolean, col 47:decimal(10,2), null)(children: StringGroupColEqualStringScalar(col 23:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 18:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 47:decimal(10,2)) -> 31:decimal(10,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 23:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 19:boolean) -> 32:decimal(12,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 23:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 19:boolean) -> 33:decimal(12,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 23:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 19:boolean) -> 34:decimal(10,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 23:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 19:boolean) -> 35:decimal(10,2), IfExprTimestampColumnColumn(col 19:boolean, col 36:timestampcol 37:timestamp)(children: LongColGreaterLongScalar(col 1:int, val 30) -> 19:boolean, CastDateToTimestamp(col 12:date) -> 36:timestamp, CastDateToTimestamp(col 11:date) -> 37:timestamp) -> 38:timestamp, IfExprColumnNull(col 19:boolean, col 39:int, null)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 19:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 39:int) -> 40:int, IfExprNullColumn(col 41:boolean, null, col 42)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 41:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 42:int) -> 43:int, IfExprLongScalarLongScalar(col 45:boolean, val 14245, val 14609)(children: LongColGreaterLongScalar(col 44:int, val 100)(children: LongColModuloLongScalar(col 2:int, val 500) -> 44:int) -> 45:boolean) -> 44:date + projectedOutputColumnNums: [4, 22, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 36, 40, 42, 45, 46] + selectExpressions: IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 21:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 22:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 21:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprStringScalarStringScalar(col 20:boolean, val Many, val Huge number)(children: LongColLessLongScalar(col 4:int, val 100) -> 20:boolean) -> 21:string) -> 22:string) -> 21:string) -> 22:string, IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 23:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 24:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 23:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprColumnNull(col 20:boolean, col 21:string, null)(children: LongColLessLongScalar(col 4:int, val 100) -> 20:boolean, ConstantVectorExpression(val Many) -> 21:string) -> 23:string) -> 24:string) -> 23:string) -> 24:string, IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 23:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 25:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 23:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprNullNull(null, null) -> 23:string) -> 25:string) -> 23:string) -> 25:string, IfExprLongColumnLongColumn(col 17:boolean, col 18:date, col 19:date)(children: StringGroupColEqualCharScalar(col 14:char(10), val SHIP) -> 17:boolean, VectorUDFDateAddColScalar(col 10:date, val 10) -> 18:date, VectorUDFDateAddColScalar(col 10:date, val 5) -> 19:date) -> 26:date, IfExprDoubleColumnLongScalar(col 17:boolean, col 28:double, val 0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 27:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 27:double) -> 28:double) -> 27:double, IfExprDoubleColumnDoubleScalar(col 17:boolean, col 29:double, val 0.0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 28:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 28:double) -> 29:double) -> 28:double, IfExprNullColumn(col 17:boolean, null, col 48)(children: StringGroupColEqualStringScalar(col 23:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 17:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 48:decimal(10,2)) -> 30:decimal(10,2), IfExprColumnNull(col 18:boolean, col 49:decimal(10,2), null)(children: StringGroupColEqualStringScalar(col 23:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 18:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 49:decimal(10,2)) -> 31:decimal(10,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 23:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 19:boolean) -> 32:decimal(12,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 23:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 19:boolean) -> 33:decimal(12,2), IfExprDecimal64ScalarDecimal64Column(col 19:boolean, decimal64Val 0, decimalVal 0, col 7:decimal(1,0)/DECIMAL_64)(children: StringGroupColEqualStringScalar(col 23:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 19:boolean) -> 34:decimal(10,2)/DECIMAL_64, IfExprDecimal64ColumnDecimal64Scalar(col 35:boolean, col 7:decimal(10,2)/DECIMAL_64, decimal64Val 0, decimalVal 0)(children: StringGroupColEqualStringScalar(col 23:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 35:boolean) -> 36:decimal(10,2)/DECIMAL_64, IfExprTimestampColumnColumn(col 37:boolean, col 38:timestampcol 39:timestamp)(children: LongColGreaterLongScalar(col 1:int, val 30) -> 37:boolean, CastDateToTimestamp(col 12:date) -> 38:timestamp, CastDateToTimestamp(col 11:date) -> 39:timestamp) -> 40:timestamp, IfExprColumnNull(col 37:boolean, col 41:int, null)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 37:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 41:int) -> 42:int, IfExprNullColumn(col 43:boolean, null, col 44)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 43:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 44:int) -> 45:int, IfExprLongScalarLongScalar(col 47:boolean, val 14245, val 14609)(children: LongColGreaterLongScalar(col 46:int, val 100)(children: LongColModuloLongScalar(col 2:int, val 500) -> 46:int) -> 47:boolean) -> 46:date Statistics: Num rows: 101 Data size: 57327 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -552,7 +552,7 @@ STAGE PLANS: includeColumns: [1, 2, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14] dataColumns: l_orderkey:int, l_partkey:int, l_suppkey:int, l_linenumber:int, l_quantity:int, l_extendedprice:double, l_discount:double, l_tax:decimal(10,2)/DECIMAL_64, l_returnflag:char(1), l_linestatus:char(1), l_shipdate:date, l_commitdate:date, l_receiptdate:date, l_shipinstruct:varchar(20), l_shipmode:char(10), l_comment:string partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, bigint, bigint, bigint, string, string, string, string, string, bigint, double, double, double, decimal(10,2), decimal(10,2), decimal(12,2), decimal(12,2), decimal(10,2), decimal(10,2), timestamp, timestamp, timestamp, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(10,2), decimal(10,2)] + scratchColumnTypeNames: [bigint, bigint, bigint, bigint, string, string, string, string, string, bigint, double, double, double, decimal(10,2), decimal(10,2), decimal(12,2), decimal(12,2), decimal(10,2)/DECIMAL_64, bigint, decimal(10,2)/DECIMAL_64, bigint, timestamp, timestamp, timestamp, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(10,2), decimal(10,2)] Stage: Stage-0 Fetch Operator @@ -856,8 +856,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [4, 27, 38, 48, 52, 54, 60, 63, 65, 67, 68, 69, 70, 73, 76, 79, 80] - selectExpressions: IfExprColumnCondExpr(col 17:boolean, col 18:stringcol 26:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, ConstantVectorExpression(val Single) -> 18:string, IfExprColumnCondExpr(col 19:boolean, col 20:stringcol 25:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 19:boolean, ConstantVectorExpression(val Two) -> 20:string, IfExprColumnCondExpr(col 21:boolean, col 22:stringcol 24:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 21:boolean, ConstantVectorExpression(val Some) -> 22:string, IfExprStringScalarStringScalar(col 23:boolean, val Many, val Huge number)(children: LongColLessLongScalar(col 4:int, val 100) -> 23:boolean) -> 24:string) -> 25:string) -> 26:string) -> 27:string, IfExprColumnCondExpr(col 23:boolean, col 28:stringcol 37:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 23:boolean, ConstantVectorExpression(val Single) -> 28:string, IfExprColumnCondExpr(col 29:boolean, col 30:stringcol 36:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 29:boolean, ConstantVectorExpression(val Two) -> 30:string, IfExprColumnCondExpr(col 31:boolean, col 32:stringcol 35:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 31:boolean, ConstantVectorExpression(val Some) -> 32:string, IfExprColumnNull(col 33:boolean, col 34:string, null)(children: LongColLessLongScalar(col 4:int, val 100) -> 33:boolean, ConstantVectorExpression(val Many) -> 34:string) -> 35:string) -> 36:string) -> 37:string) -> 38:string, IfExprColumnCondExpr(col 39:boolean, col 40:stringcol 47:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 39:boolean, ConstantVectorExpression(val Single) -> 40:string, IfExprColumnCondExpr(col 41:boolean, col 42:stringcol 46:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 41:boolean, ConstantVectorExpression(val Two) -> 42:string, IfExprColumnCondExpr(col 43:boolean, col 44:stringcol 45:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 43:boolean, ConstantVectorExpression(val Some) -> 44:string, IfExprNullNull(null, null) -> 45:string) -> 46:string) -> 47:string) -> 48:string, IfExprCondExprCondExpr(col 49:boolean, col 50:datecol 51:date)(children: StringGroupColEqualCharScalar(col 14:char(10), val SHIP) -> 49:boolean, VectorUDFDateAddColScalar(col 10:date, val 10) -> 50:date, VectorUDFDateAddColScalar(col 10:date, val 5) -> 51:date) -> 52:date, IfExprDoubleColumnLongScalar(col 57:boolean, col 58:double, val 0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 57:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 54:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 54:double) -> 58:double) -> 54:double, IfExprCondExprColumn(col 57:boolean, col 59:double, col 58:double)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 57:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 58:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 58:double) -> 59:double, ConstantVectorExpression(val 0.0) -> 58:double) -> 60:double, IfExprNullColumn(col 62:boolean, null, col 82)(children: StringGroupColEqualStringScalar(col 61:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 62:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 82:decimal(10,2)) -> 63:decimal(10,2), IfExprColumnNull(col 64:boolean, col 83:decimal(10,2), null)(children: StringGroupColEqualStringScalar(col 61:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 64:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 83:decimal(10,2)) -> 65:decimal(10,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 61:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 66:boolean) -> 67:decimal(12,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 61:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 66:boolean) -> 68:decimal(12,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 61:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 66:boolean) -> 69:decimal(10,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 61:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 66:boolean) -> 70:decimal(10,2), IfExprCondExprCondExpr(col 66:boolean, col 71:timestampcol 72:timestamp)(children: LongColGreaterLongScalar(col 1:int, val 30) -> 66:boolean, CastDateToTimestamp(col 12:date) -> 71:timestamp, CastDateToTimestamp(col 11:date) -> 72:timestamp) -> 73:timestamp, IfExprCondExprNull(col 74:boolean, col 75:int, null)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 74:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 75:int) -> 76:int, IfExprNullCondExpr(col 77:boolean, null, col 78:int)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 77:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 78:int) -> 79:int, IfExprLongScalarLongScalar(col 81:boolean, val 14245, val 14609)(children: LongColGreaterLongScalar(col 80:int, val 100)(children: LongColModuloLongScalar(col 2:int, val 500) -> 80:int) -> 81:boolean) -> 80:date + projectedOutputColumnNums: [4, 27, 38, 48, 52, 54, 60, 63, 65, 67, 68, 69, 71, 75, 78, 81, 82] + selectExpressions: IfExprColumnCondExpr(col 17:boolean, col 18:stringcol 26:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, ConstantVectorExpression(val Single) -> 18:string, IfExprColumnCondExpr(col 19:boolean, col 20:stringcol 25:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 19:boolean, ConstantVectorExpression(val Two) -> 20:string, IfExprColumnCondExpr(col 21:boolean, col 22:stringcol 24:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 21:boolean, ConstantVectorExpression(val Some) -> 22:string, IfExprStringScalarStringScalar(col 23:boolean, val Many, val Huge number)(children: LongColLessLongScalar(col 4:int, val 100) -> 23:boolean) -> 24:string) -> 25:string) -> 26:string) -> 27:string, IfExprColumnCondExpr(col 23:boolean, col 28:stringcol 37:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 23:boolean, ConstantVectorExpression(val Single) -> 28:string, IfExprColumnCondExpr(col 29:boolean, col 30:stringcol 36:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 29:boolean, ConstantVectorExpression(val Two) -> 30:string, IfExprColumnCondExpr(col 31:boolean, col 32:stringcol 35:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 31:boolean, ConstantVectorExpression(val Some) -> 32:string, IfExprColumnNull(col 33:boolean, col 34:string, null)(children: LongColLessLongScalar(col 4:int, val 100) -> 33:boolean, ConstantVectorExpression(val Many) -> 34:string) -> 35:string) -> 36:string) -> 37:string) -> 38:string, IfExprColumnCondExpr(col 39:boolean, col 40:stringcol 47:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 39:boolean, ConstantVectorExpression(val Single) -> 40:string, IfExprColumnCondExpr(col 41:boolean, col 42:stringcol 46:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 41:boolean, ConstantVectorExpression(val Two) -> 42:string, IfExprColumnCondExpr(col 43:boolean, col 44:stringcol 45:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 43:boolean, ConstantVectorExpression(val Some) -> 44:string, IfExprNullNull(null, null) -> 45:string) -> 46:string) -> 47:string) -> 48:string, IfExprCondExprCondExpr(col 49:boolean, col 50:datecol 51:date)(children: StringGroupColEqualCharScalar(col 14:char(10), val SHIP) -> 49:boolean, VectorUDFDateAddColScalar(col 10:date, val 10) -> 50:date, VectorUDFDateAddColScalar(col 10:date, val 5) -> 51:date) -> 52:date, IfExprDoubleColumnLongScalar(col 57:boolean, col 58:double, val 0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 57:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 54:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 54:double) -> 58:double) -> 54:double, IfExprCondExprColumn(col 57:boolean, col 59:double, col 58:double)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 57:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 58:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 58:double) -> 59:double, ConstantVectorExpression(val 0.0) -> 58:double) -> 60:double, IfExprNullColumn(col 62:boolean, null, col 84)(children: StringGroupColEqualStringScalar(col 61:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 62:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 84:decimal(10,2)) -> 63:decimal(10,2), IfExprColumnNull(col 64:boolean, col 85:decimal(10,2), null)(children: StringGroupColEqualStringScalar(col 61:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 64:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 85:decimal(10,2)) -> 65:decimal(10,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 61:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 66:boolean) -> 67:decimal(12,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 61:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 66:boolean) -> 68:decimal(12,2), IfExprDecimal64ScalarDecimal64Column(col 66:boolean, decimal64Val 0, decimalVal 0, col 7:decimal(1,0)/DECIMAL_64)(children: StringGroupColEqualStringScalar(col 61:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 66:boolean) -> 69:decimal(10,2)/DECIMAL_64, IfExprDecimal64ColumnDecimal64Scalar(col 70:boolean, col 7:decimal(10,2)/DECIMAL_64, decimal64Val 0, decimalVal 0)(children: StringGroupColEqualStringScalar(col 61:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 70:boolean) -> 71:decimal(10,2)/DECIMAL_64, IfExprCondExprCondExpr(col 72:boolean, col 73:timestampcol 74:timestamp)(children: LongColGreaterLongScalar(col 1:int, val 30) -> 72:boolean, CastDateToTimestamp(col 12:date) -> 73:timestamp, CastDateToTimestamp(col 11:date) -> 74:timestamp) -> 75:timestamp, IfExprCondExprNull(col 76:boolean, col 77:int, null)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 76:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 77:int) -> 78:int, IfExprNullCondExpr(col 79:boolean, null, col 80:int)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 79:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 80:int) -> 81:int, IfExprLongScalarLongScalar(col 83:boolean, val 14245, val 14609)(children: LongColGreaterLongScalar(col 82:int, val 100)(children: LongColModuloLongScalar(col 2:int, val 500) -> 82:int) -> 83:boolean) -> 82:date Statistics: Num rows: 101 Data size: 57327 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -885,7 +885,7 @@ STAGE PLANS: includeColumns: [1, 2, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14] dataColumns: l_orderkey:int, l_partkey:int, l_suppkey:int, l_linenumber:int, l_quantity:int, l_extendedprice:double, l_discount:double, l_tax:decimal(10,2)/DECIMAL_64, l_returnflag:char(1), l_linestatus:char(1), l_shipdate:date, l_commitdate:date, l_receiptdate:date, l_shipinstruct:varchar(20), l_shipmode:char(10), l_comment:string partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, bigint, bigint, bigint, bigint, double, double, bigint, bigint, double, double, double, string, bigint, decimal(10,2), bigint, decimal(10,2), bigint, decimal(12,2), decimal(12,2), decimal(10,2), decimal(10,2), timestamp, timestamp, timestamp, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(10,2), decimal(10,2)] + scratchColumnTypeNames: [bigint, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, bigint, bigint, bigint, bigint, double, double, bigint, bigint, double, double, double, string, bigint, decimal(10,2), bigint, decimal(10,2), bigint, decimal(12,2), decimal(12,2), decimal(10,2)/DECIMAL_64, bigint, decimal(10,2)/DECIMAL_64, bigint, timestamp, timestamp, timestamp, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(10,2), decimal(10,2)] Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_mapjoin.q.out ql/src/test/results/clientpositive/llap/vector_decimal_mapjoin.q.out index 27d6c1c..b66fb9f 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_mapjoin.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_mapjoin.q.out @@ -740,7 +740,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(14,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(14,2)/DECIMAL_64) -> 3:decimal(14,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(14,2)/DECIMAL_64) predicate: dec is not null (type: boolean) Statistics: Num rows: 997 Data size: 106235 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -758,8 +758,8 @@ STAGE PLANS: 0 _col0 (type: decimal(16,2)) 1 _col0 (type: decimal(16,2)) Map Join Vectorization: - bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 4:decimal(16,2) - bigTableValueExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 5:decimal(16,2) + bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 3:decimal(16,2) + bigTableValueExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 4:decimal(16,2) className: VectorMapJoinOperator native: false nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true @@ -795,7 +795,7 @@ STAGE PLANS: includeColumns: [0] dataColumns: dec:decimal(14,2)/DECIMAL_64, value_dec:decimal(14,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(14,2), decimal(16,2), decimal(16,2)] + scratchColumnTypeNames: [decimal(16,2), decimal(16,2)] Map 2 Map Operator Tree: TableScan @@ -809,7 +809,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(14,0))(children: ConvertDecimal64ToDecimal(col 0:decimal(14,0)/DECIMAL_64) -> 3:decimal(14,0)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(14,0)/DECIMAL_64) predicate: dec is not null (type: boolean) Statistics: Num rows: 997 Data size: 106235 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -847,7 +847,7 @@ STAGE PLANS: includeColumns: [0] dataColumns: dec:decimal(14,0)/DECIMAL_64, value_dec:decimal(14,0)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(14,0)] + scratchColumnTypeNames: [] Stage: Stage-0 Fetch Operator @@ -1017,7 +1017,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(14,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(14,2)/DECIMAL_64) -> 3:decimal(14,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(14,2)/DECIMAL_64) predicate: dec is not null (type: boolean) Statistics: Num rows: 997 Data size: 212470 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -1035,8 +1035,8 @@ STAGE PLANS: 0 _col0 (type: decimal(16,2)) 1 _col0 (type: decimal(16,2)) Map Join Vectorization: - bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 4:decimal(16,2) - bigTableValueExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 5:decimal(16,2), ConvertDecimal64ToDecimal(col 1:decimal(14,2)/DECIMAL_64) -> 3:decimal(14,2) + bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 3:decimal(16,2) + bigTableValueExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 4:decimal(16,2), ConvertDecimal64ToDecimal(col 1:decimal(14,2)/DECIMAL_64) -> 5:decimal(14,2) className: VectorMapJoinOperator native: false nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true @@ -1072,7 +1072,7 @@ STAGE PLANS: includeColumns: [0, 1] dataColumns: dec:decimal(14,2)/DECIMAL_64, value_dec:decimal(14,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(14,2), decimal(16,2), decimal(16,2), decimal(14,0)] + scratchColumnTypeNames: [decimal(16,2), decimal(16,2), decimal(14,2), decimal(14,0)] Map 2 Map Operator Tree: TableScan @@ -1086,7 +1086,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(14,0))(children: ConvertDecimal64ToDecimal(col 0:decimal(14,0)/DECIMAL_64) -> 3:decimal(14,0)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(14,0)/DECIMAL_64) predicate: dec is not null (type: boolean) Statistics: Num rows: 997 Data size: 212470 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -1125,7 +1125,7 @@ STAGE PLANS: includeColumns: [0, 1] dataColumns: dec:decimal(14,0)/DECIMAL_64, value_dec:decimal(14,0)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(14,0)] + scratchColumnTypeNames: [] Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_outer_reference_windowed.q.out ql/src/test/results/clientpositive/llap/vector_outer_reference_windowed.q.out index 897c145..ba2d9df 100644 --- ql/src/test/results/clientpositive/llap/vector_outer_reference_windowed.q.out +++ ql/src/test/results/clientpositive/llap/vector_outer_reference_windowed.q.out @@ -730,7 +730,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(15,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(15,2)/DECIMAL_64) -> 3:decimal(15,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(15,2)/DECIMAL_64) predicate: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 896 Basic stats: COMPLETE Column stats: COMPLETE Select Operator @@ -769,7 +769,7 @@ STAGE PLANS: includeColumns: [0, 1] dataColumns: c1:decimal(15,2)/DECIMAL_64, c2:decimal(15,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(15,2)] + scratchColumnTypeNames: [] Map 5 Map Operator Tree: TableScan @@ -783,7 +783,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(15,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(15,2)/DECIMAL_64) -> 3:decimal(15,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(15,2)/DECIMAL_64) predicate: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE Select Operator @@ -821,7 +821,7 @@ STAGE PLANS: includeColumns: [0] dataColumns: c1:decimal(15,2)/DECIMAL_64, c2:decimal(15,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(15,2)] + scratchColumnTypeNames: [] Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -1039,7 +1039,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(15,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(15,2)/DECIMAL_64) -> 3:decimal(15,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(15,2)/DECIMAL_64) predicate: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE Select Operator @@ -1077,7 +1077,7 @@ STAGE PLANS: includeColumns: [0] dataColumns: c1:decimal(15,2)/DECIMAL_64, c2:decimal(15,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(15,2)] + scratchColumnTypeNames: [] Map 5 Map Operator Tree: TableScan @@ -1091,7 +1091,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(15,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(15,2)/DECIMAL_64) -> 3:decimal(15,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(15,2)/DECIMAL_64) predicate: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 896 Basic stats: COMPLETE Column stats: COMPLETE Select Operator @@ -1130,7 +1130,7 @@ STAGE PLANS: includeColumns: [0, 1] dataColumns: c1:decimal(15,2)/DECIMAL_64, c2:decimal(15,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(15,2)] + scratchColumnTypeNames: [] Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -1347,7 +1347,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(15,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(15,2)/DECIMAL_64) -> 3:decimal(15,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(15,2)/DECIMAL_64) predicate: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 896 Basic stats: COMPLETE Column stats: COMPLETE Select Operator @@ -1386,7 +1386,7 @@ STAGE PLANS: includeColumns: [0, 1] dataColumns: c1:decimal(15,2)/DECIMAL_64, c2:decimal(15,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(15,2)] + scratchColumnTypeNames: [] Map 4 Map Operator Tree: TableScan @@ -1400,7 +1400,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(15,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(15,2)/DECIMAL_64) -> 3:decimal(15,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(15,2)/DECIMAL_64) predicate: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 896 Basic stats: COMPLETE Column stats: COMPLETE Select Operator @@ -1439,7 +1439,7 @@ STAGE PLANS: includeColumns: [0, 1] dataColumns: c1:decimal(15,2)/DECIMAL_64, c2:decimal(15,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(15,2)] + scratchColumnTypeNames: [] Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -2028,7 +2028,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(7,2)/DECIMAL_64) -> 3:decimal(7,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(7,2)/DECIMAL_64) predicate: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 896 Basic stats: COMPLETE Column stats: COMPLETE Select Operator @@ -2067,7 +2067,7 @@ STAGE PLANS: includeColumns: [0, 1] dataColumns: c1:decimal(7,2)/DECIMAL_64, c2:decimal(7,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(7,2)] + scratchColumnTypeNames: [] Map 5 Map Operator Tree: TableScan @@ -2081,7 +2081,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(7,2)/DECIMAL_64) -> 3:decimal(7,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(7,2)/DECIMAL_64) predicate: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE Select Operator @@ -2119,7 +2119,7 @@ STAGE PLANS: includeColumns: [0] dataColumns: c1:decimal(7,2)/DECIMAL_64, c2:decimal(7,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(7,2)] + scratchColumnTypeNames: [] Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -2337,7 +2337,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(7,2)/DECIMAL_64) -> 3:decimal(7,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(7,2)/DECIMAL_64) predicate: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE Select Operator @@ -2375,7 +2375,7 @@ STAGE PLANS: includeColumns: [0] dataColumns: c1:decimal(7,2)/DECIMAL_64, c2:decimal(7,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(7,2)] + scratchColumnTypeNames: [] Map 5 Map Operator Tree: TableScan @@ -2389,7 +2389,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(7,2)/DECIMAL_64) -> 3:decimal(7,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(7,2)/DECIMAL_64) predicate: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 896 Basic stats: COMPLETE Column stats: COMPLETE Select Operator @@ -2428,7 +2428,7 @@ STAGE PLANS: includeColumns: [0, 1] dataColumns: c1:decimal(7,2)/DECIMAL_64, c2:decimal(7,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(7,2)] + scratchColumnTypeNames: [] Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -2645,7 +2645,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(7,2)/DECIMAL_64) -> 3:decimal(7,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(7,2)/DECIMAL_64) predicate: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 896 Basic stats: COMPLETE Column stats: COMPLETE Select Operator @@ -2684,7 +2684,7 @@ STAGE PLANS: includeColumns: [0, 1] dataColumns: c1:decimal(7,2)/DECIMAL_64, c2:decimal(7,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(7,2)] + scratchColumnTypeNames: [] Map 4 Map Operator Tree: TableScan @@ -2698,7 +2698,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(7,2)/DECIMAL_64) -> 3:decimal(7,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(7,2)/DECIMAL_64) predicate: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 896 Basic stats: COMPLETE Column stats: COMPLETE Select Operator @@ -2737,7 +2737,7 @@ STAGE PLANS: includeColumns: [0, 1] dataColumns: c1:decimal(7,2)/DECIMAL_64, c2:decimal(7,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(7,2)] + scratchColumnTypeNames: [] Reducer 2 Execution mode: llap Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/vectorized_case.q.out ql/src/test/results/clientpositive/llap/vectorized_case.q.out index 78098b4..7e5fee0 100644 --- ql/src/test/results/clientpositive/llap/vectorized_case.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_case.q.out @@ -596,7 +596,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [6] - selectExpressions: IfExprCondExprCondExpr(col 3:boolean, col 7:decimal(11,0)col 8:decimal(11,0))(children: VectorUDFAdaptor((member = 1)) -> 3:boolean, ConvertDecimal64ToDecimal(col 4:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 4:decimal(11,0)/DECIMAL_64) -> 7:decimal(11,0), ConvertDecimal64ToDecimal(col 5:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 2, decimalVal 2) -> 5:decimal(11,0)/DECIMAL_64) -> 8:decimal(11,0)) -> 6:decimal(11,0) + selectExpressions: IfExprCondExprCondExpr(col 3:boolean, col 7:decimal(11,0)col 8:decimal(11,0))(children: Decimal64ColEqualDecimal64Scalar(col 0:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 3:boolean, ConvertDecimal64ToDecimal(col 4:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 4:decimal(11,0)/DECIMAL_64) -> 7:decimal(11,0), ConvertDecimal64ToDecimal(col 5:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 2, decimalVal 2) -> 5:decimal(11,0)/DECIMAL_64) -> 8:decimal(11,0)) -> 6:decimal(11,0) Statistics: Num rows: 3 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -617,7 +617,7 @@ STAGE PLANS: featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 2 @@ -677,7 +677,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [8] - selectExpressions: VectorUDFAdaptor(CASE WHEN ((member = 1)) THEN (1) ELSE ((attr + 2)) END)(children: VectorUDFAdaptor((member = 1)) -> 6:boolean, ConvertDecimal64ToDecimal(col 7:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 2, decimalVal 2) -> 7:decimal(11,0)/DECIMAL_64) -> 9:decimal(11,0)) -> 8:decimal(11,0) + selectExpressions: IfExprDecimal64ScalarDecimal64Column(col 6:boolean, decimal64Val 1, decimalVal 1, col 7:decimal(1,0)/DECIMAL_64)(children: Decimal64ColEqualDecimal64Scalar(col 0:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 6:boolean, Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 2, decimalVal 2) -> 7:decimal(11,0)/DECIMAL_64) -> 8:decimal(11,0)/DECIMAL_64 Statistics: Num rows: 3 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -698,14 +698,14 @@ STAGE PLANS: featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 2 includeColumns: [0, 1] dataColumns: member:decimal(10,0)/DECIMAL_64, attr:decimal(10,0)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, decimal(1,0), decimal(11,0)/DECIMAL_64, bigint, decimal(11,0)/DECIMAL_64, decimal(11,0), decimal(11,0)] + scratchColumnTypeNames: [bigint, decimal(1,0), decimal(11,0)/DECIMAL_64, bigint, decimal(11,0)/DECIMAL_64, decimal(11,0)/DECIMAL_64] Stage: Stage-0 Fetch Operator @@ -758,7 +758,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [8] - selectExpressions: VectorUDFAdaptor(CASE WHEN ((member = 1)) THEN ((attr + 1)) ELSE (2) END)(children: VectorUDFAdaptor((member = 1)) -> 6:boolean, ConvertDecimal64ToDecimal(col 7:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 7:decimal(11,0)/DECIMAL_64) -> 9:decimal(11,0)) -> 8:decimal(11,0) + selectExpressions: IfExprDecimal64ColumnDecimal64Scalar(col 6:boolean, col 7:decimal(11,0)/DECIMAL_64, decimal64Val 2, decimalVal 2)(children: Decimal64ColEqualDecimal64Scalar(col 0:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 6:boolean, Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 7:decimal(11,0)/DECIMAL_64) -> 8:decimal(11,0)/DECIMAL_64 Statistics: Num rows: 3 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -779,14 +779,14 @@ STAGE PLANS: featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 2 includeColumns: [0, 1] dataColumns: member:decimal(10,0)/DECIMAL_64, attr:decimal(10,0)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, decimal(11,0)/DECIMAL_64, decimal(1,0), bigint, decimal(11,0)/DECIMAL_64, decimal(11,0), decimal(11,0)] + scratchColumnTypeNames: [bigint, decimal(11,0)/DECIMAL_64, decimal(1,0), bigint, decimal(11,0)/DECIMAL_64, decimal(11,0)/DECIMAL_64] Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out index c03b298..e416954 100644 --- ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out @@ -262,7 +262,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: SelectColumnIsNotNull(col 9:decimal(10,1))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,1)/DECIMAL_64) -> 9:decimal(10,1)), FilterExprAndExpr(children: FilterDecimalColumnBetweenDynamicValue(col 9:decimal(10,1), left 0, right 0)(children: ConvertDecimal64ToDecimal(col 1:decimal(10,1)/DECIMAL_64) -> 9:decimal(10,1)), VectorInBloomFilterColDynamicValue(children: ConvertDecimal64ToDecimal(col 1:decimal(10,1)/DECIMAL_64) -> 9:decimal(10,1)))) + predicateExpression: FilterExprAndExpr(children: SelectColumnIsNotNull(col 1:decimal(10,1)/DECIMAL_64), FilterExprAndExpr(children: FilterDecimalColumnBetweenDynamicValue(col 9:decimal(10,1), left 0, right 0)(children: ConvertDecimal64ToDecimal(col 1:decimal(10,1)/DECIMAL_64) -> 9:decimal(10,1)), VectorInBloomFilterColDynamicValue(children: ConvertDecimal64ToDecimal(col 1:decimal(10,1)/DECIMAL_64) -> 9:decimal(10,1)))) predicate: ((partkey_decimal BETWEEN DynamicValue(RS_7_b_partkey_decimal_min) AND DynamicValue(RS_7_b_partkey_decimal_max) and in_bloom_filter(partkey_decimal, DynamicValue(RS_7_b_partkey_decimal_bloom_filter))) and partkey_decimal is not null) (type: boolean) Statistics: Num rows: 100 Data size: 11200 Basic stats: COMPLETE Column stats: COMPLETE Select Operator @@ -314,7 +314,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 9:decimal(10,1))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,1)/DECIMAL_64) -> 9:decimal(10,1)) + predicateExpression: SelectColumnIsNotNull(col 1:decimal(10,1)/DECIMAL_64) predicate: partkey_decimal is not null (type: boolean) Statistics: Num rows: 20 Data size: 2240 Basic stats: COMPLETE Column stats: COMPLETE Select Operator diff --git ql/src/test/results/clientpositive/llap/vectorized_mapjoin3.q.out ql/src/test/results/clientpositive/llap/vectorized_mapjoin3.q.out index e9aea17..23120a5 100644 --- ql/src/test/results/clientpositive/llap/vectorized_mapjoin3.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_mapjoin3.q.out @@ -128,7 +128,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: SelectColumnIsNotNull(col 3:decimal(8,1))(children: ConvertDecimal64ToDecimal(col 0:decimal(8,1)/DECIMAL_64) -> 3:decimal(8,1)), SelectColumnIsNotNull(col 1:int)) + predicateExpression: FilterExprAndExpr(children: SelectColumnIsNotNull(col 0:decimal(8,1)/DECIMAL_64), SelectColumnIsNotNull(col 1:int)) predicate: (decimal0801_col is not null and int_col_1 is not null) (type: boolean) Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE Select Operator @@ -333,7 +333,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: SelectColumnIsNotNull(col 1:int), SelectColumnIsNotNull(col 3:decimal(8,1))(children: ConvertDecimal64ToDecimal(col 0:decimal(8,1)/DECIMAL_64) -> 3:decimal(8,1))) + predicateExpression: FilterExprAndExpr(children: SelectColumnIsNotNull(col 1:int), SelectColumnIsNotNull(col 0:decimal(8,1)/DECIMAL_64)) predicate: (decimal0801_col is not null and int_col_1 is not null) (type: boolean) Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE Select Operator @@ -538,7 +538,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: SelectColumnIsNotNull(col 3:decimal(8,1))(children: ConvertDecimal64ToDecimal(col 0:decimal(8,1)/DECIMAL_64) -> 3:decimal(8,1)), SelectColumnIsNotNull(col 1:int)) + predicateExpression: FilterExprAndExpr(children: SelectColumnIsNotNull(col 0:decimal(8,1)/DECIMAL_64), SelectColumnIsNotNull(col 1:int)) predicate: (decimal0801_col is not null and int_col_1 is not null) (type: boolean) Statistics: Num rows: 4 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE Select Operator diff --git ql/src/test/results/clientpositive/spark/vector_decimal_mapjoin.q.out ql/src/test/results/clientpositive/spark/vector_decimal_mapjoin.q.out index f711413..c15b3b9 100644 --- ql/src/test/results/clientpositive/spark/vector_decimal_mapjoin.q.out +++ ql/src/test/results/clientpositive/spark/vector_decimal_mapjoin.q.out @@ -739,7 +739,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(14,0))(children: ConvertDecimal64ToDecimal(col 0:decimal(14,0)/DECIMAL_64) -> 3:decimal(14,0)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(14,0)/DECIMAL_64) predicate: dec is not null (type: boolean) Statistics: Num rows: 1049 Data size: 4966 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -772,7 +772,7 @@ STAGE PLANS: includeColumns: [0] dataColumns: dec:decimal(14,0)/DECIMAL_64, value_dec:decimal(14,0)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(14,0)] + scratchColumnTypeNames: [] Local Work: Map Reduce Local Work @@ -793,7 +793,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(14,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(14,2)/DECIMAL_64) -> 3:decimal(14,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(14,2)/DECIMAL_64) predicate: dec is not null (type: boolean) Statistics: Num rows: 1049 Data size: 11234 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -811,8 +811,8 @@ STAGE PLANS: 0 _col0 (type: decimal(16,2)) 1 _col0 (type: decimal(16,2)) Map Join Vectorization: - bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 4:decimal(16,2) - bigTableValueExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 5:decimal(16,2) + bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 3:decimal(16,2) + bigTableValueExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 4:decimal(16,2) className: VectorMapJoinOperator native: false nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true @@ -847,7 +847,7 @@ STAGE PLANS: includeColumns: [0] dataColumns: dec:decimal(14,2)/DECIMAL_64, value_dec:decimal(14,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(14,2), decimal(16,2), decimal(16,2)] + scratchColumnTypeNames: [decimal(16,2), decimal(16,2)] Local Work: Map Reduce Local Work @@ -1017,7 +1017,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(14,0))(children: ConvertDecimal64ToDecimal(col 0:decimal(14,0)/DECIMAL_64) -> 3:decimal(14,0)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(14,0)/DECIMAL_64) predicate: dec is not null (type: boolean) Statistics: Num rows: 1049 Data size: 4966 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -1050,7 +1050,7 @@ STAGE PLANS: includeColumns: [0, 1] dataColumns: dec:decimal(14,0)/DECIMAL_64, value_dec:decimal(14,0)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(14,0)] + scratchColumnTypeNames: [] Local Work: Map Reduce Local Work @@ -1071,7 +1071,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(14,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(14,2)/DECIMAL_64) -> 3:decimal(14,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(14,2)/DECIMAL_64) predicate: dec is not null (type: boolean) Statistics: Num rows: 1049 Data size: 11234 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -1089,8 +1089,8 @@ STAGE PLANS: 0 _col0 (type: decimal(16,2)) 1 _col0 (type: decimal(16,2)) Map Join Vectorization: - bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 4:decimal(16,2) - bigTableValueExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 5:decimal(16,2), ConvertDecimal64ToDecimal(col 1:decimal(14,2)/DECIMAL_64) -> 3:decimal(14,2) + bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 3:decimal(16,2) + bigTableValueExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 4:decimal(16,2), ConvertDecimal64ToDecimal(col 1:decimal(14,2)/DECIMAL_64) -> 5:decimal(14,2) className: VectorMapJoinOperator native: false nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true @@ -1125,7 +1125,7 @@ STAGE PLANS: includeColumns: [0, 1] dataColumns: dec:decimal(14,2)/DECIMAL_64, value_dec:decimal(14,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(14,2), decimal(16,2), decimal(16,2), decimal(14,0)] + scratchColumnTypeNames: [decimal(16,2), decimal(16,2), decimal(14,2), decimal(14,0)] Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/spark/vectorized_case.q.out ql/src/test/results/clientpositive/spark/vectorized_case.q.out index 028351b..4d322cf 100644 --- ql/src/test/results/clientpositive/spark/vectorized_case.q.out +++ ql/src/test/results/clientpositive/spark/vectorized_case.q.out @@ -590,7 +590,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [6] - selectExpressions: IfExprCondExprCondExpr(col 3:boolean, col 7:decimal(11,0)col 8:decimal(11,0))(children: VectorUDFAdaptor((member = 1)) -> 3:boolean, ConvertDecimal64ToDecimal(col 4:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 4:decimal(11,0)/DECIMAL_64) -> 7:decimal(11,0), ConvertDecimal64ToDecimal(col 5:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 2, decimalVal 2) -> 5:decimal(11,0)/DECIMAL_64) -> 8:decimal(11,0)) -> 6:decimal(11,0) + selectExpressions: IfExprCondExprCondExpr(col 3:boolean, col 7:decimal(11,0)col 8:decimal(11,0))(children: Decimal64ColEqualDecimal64Scalar(col 0:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 3:boolean, ConvertDecimal64ToDecimal(col 4:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 4:decimal(11,0)/DECIMAL_64) -> 7:decimal(11,0), ConvertDecimal64ToDecimal(col 5:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 2, decimalVal 2) -> 5:decimal(11,0)/DECIMAL_64) -> 8:decimal(11,0)) -> 6:decimal(11,0) Statistics: Num rows: 3 Data size: 672 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -610,7 +610,7 @@ STAGE PLANS: featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 2 @@ -670,7 +670,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [8] - selectExpressions: VectorUDFAdaptor(CASE WHEN ((member = 1)) THEN (1) ELSE ((attr + 2)) END)(children: VectorUDFAdaptor((member = 1)) -> 6:boolean, ConvertDecimal64ToDecimal(col 7:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 2, decimalVal 2) -> 7:decimal(11,0)/DECIMAL_64) -> 9:decimal(11,0)) -> 8:decimal(11,0) + selectExpressions: IfExprDecimal64ScalarDecimal64Column(col 6:boolean, decimal64Val 1, decimalVal 1, col 7:decimal(1,0)/DECIMAL_64)(children: Decimal64ColEqualDecimal64Scalar(col 0:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 6:boolean, Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 2, decimalVal 2) -> 7:decimal(11,0)/DECIMAL_64) -> 8:decimal(11,0)/DECIMAL_64 Statistics: Num rows: 3 Data size: 672 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -690,14 +690,14 @@ STAGE PLANS: featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 2 includeColumns: [0, 1] dataColumns: member:decimal(10,0)/DECIMAL_64, attr:decimal(10,0)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, decimal(1,0), decimal(11,0)/DECIMAL_64, bigint, decimal(11,0)/DECIMAL_64, decimal(11,0), decimal(11,0)] + scratchColumnTypeNames: [bigint, decimal(1,0), decimal(11,0)/DECIMAL_64, bigint, decimal(11,0)/DECIMAL_64, decimal(11,0)/DECIMAL_64] Stage: Stage-0 Fetch Operator @@ -750,7 +750,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [8] - selectExpressions: VectorUDFAdaptor(CASE WHEN ((member = 1)) THEN ((attr + 1)) ELSE (2) END)(children: VectorUDFAdaptor((member = 1)) -> 6:boolean, ConvertDecimal64ToDecimal(col 7:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 7:decimal(11,0)/DECIMAL_64) -> 9:decimal(11,0)) -> 8:decimal(11,0) + selectExpressions: IfExprDecimal64ColumnDecimal64Scalar(col 6:boolean, col 7:decimal(11,0)/DECIMAL_64, decimal64Val 2, decimalVal 2)(children: Decimal64ColEqualDecimal64Scalar(col 0:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 6:boolean, Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 7:decimal(11,0)/DECIMAL_64) -> 8:decimal(11,0)/DECIMAL_64 Statistics: Num rows: 3 Data size: 672 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -770,14 +770,14 @@ STAGE PLANS: featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 2 includeColumns: [0, 1] dataColumns: member:decimal(10,0)/DECIMAL_64, attr:decimal(10,0)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, decimal(11,0)/DECIMAL_64, decimal(1,0), bigint, decimal(11,0)/DECIMAL_64, decimal(11,0), decimal(11,0)] + scratchColumnTypeNames: [bigint, decimal(11,0)/DECIMAL_64, decimal(1,0), bigint, decimal(11,0)/DECIMAL_64, decimal(11,0)/DECIMAL_64] Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/vector_case_when_1.q.out ql/src/test/results/clientpositive/vector_case_when_1.q.out index 59d8133..e48ee5c 100644 --- ql/src/test/results/clientpositive/vector_case_when_1.q.out +++ ql/src/test/results/clientpositive/vector_case_when_1.q.out @@ -515,8 +515,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [4, 22, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 35, 38, 40, 43, 44] - selectExpressions: IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 21:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 22:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 21:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprStringScalarStringScalar(col 20:boolean, val Many, val Huge number)(children: LongColLessLongScalar(col 4:int, val 100) -> 20:boolean) -> 21:string) -> 22:string) -> 21:string) -> 22:string, IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 23:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 24:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 23:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprColumnNull(col 20:boolean, col 21:string, null)(children: LongColLessLongScalar(col 4:int, val 100) -> 20:boolean, ConstantVectorExpression(val Many) -> 21:string) -> 23:string) -> 24:string) -> 23:string) -> 24:string, IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 23:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 25:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 23:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprNullNull(null, null) -> 23:string) -> 25:string) -> 23:string) -> 25:string, IfExprLongColumnLongColumn(col 17:boolean, col 18:date, col 19:date)(children: StringGroupColEqualCharScalar(col 14:char(10), val SHIP) -> 17:boolean, VectorUDFDateAddColScalar(col 10:date, val 10) -> 18:date, VectorUDFDateAddColScalar(col 10:date, val 5) -> 19:date) -> 26:date, IfExprDoubleColumnLongScalar(col 17:boolean, col 28:double, val 0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 27:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 27:double) -> 28:double) -> 27:double, IfExprDoubleColumnDoubleScalar(col 17:boolean, col 29:double, val 0.0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 28:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 28:double) -> 29:double) -> 28:double, IfExprNullColumn(col 17:boolean, null, col 46)(children: StringGroupColEqualStringScalar(col 23:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 17:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 46:decimal(10,2)) -> 30:decimal(10,2), IfExprColumnNull(col 18:boolean, col 47:decimal(10,2), null)(children: StringGroupColEqualStringScalar(col 23:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 18:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 47:decimal(10,2)) -> 31:decimal(10,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 23:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 19:boolean) -> 32:decimal(12,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 23:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 19:boolean) -> 33:decimal(12,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 23:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 19:boolean) -> 34:decimal(10,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 23:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 19:boolean) -> 35:decimal(10,2), IfExprTimestampColumnColumn(col 19:boolean, col 36:timestampcol 37:timestamp)(children: LongColGreaterLongScalar(col 1:int, val 30) -> 19:boolean, CastDateToTimestamp(col 12:date) -> 36:timestamp, CastDateToTimestamp(col 11:date) -> 37:timestamp) -> 38:timestamp, IfExprColumnNull(col 19:boolean, col 39:int, null)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 19:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 39:int) -> 40:int, IfExprNullColumn(col 41:boolean, null, col 42)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 41:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 42:int) -> 43:int, IfExprLongScalarLongScalar(col 45:boolean, val 14245, val 14609)(children: LongColGreaterLongScalar(col 44:int, val 100)(children: LongColModuloLongScalar(col 2:int, val 500) -> 44:int) -> 45:boolean) -> 44:date + projectedOutputColumnNums: [4, 22, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 36, 40, 42, 45, 46] + selectExpressions: IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 21:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 22:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 21:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprStringScalarStringScalar(col 20:boolean, val Many, val Huge number)(children: LongColLessLongScalar(col 4:int, val 100) -> 20:boolean) -> 21:string) -> 22:string) -> 21:string) -> 22:string, IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 23:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 24:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 23:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprColumnNull(col 20:boolean, col 21:string, null)(children: LongColLessLongScalar(col 4:int, val 100) -> 20:boolean, ConstantVectorExpression(val Many) -> 21:string) -> 23:string) -> 24:string) -> 23:string) -> 24:string, IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 23:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 25:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 23:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprNullNull(null, null) -> 23:string) -> 25:string) -> 23:string) -> 25:string, IfExprLongColumnLongColumn(col 17:boolean, col 18:date, col 19:date)(children: StringGroupColEqualCharScalar(col 14:char(10), val SHIP) -> 17:boolean, VectorUDFDateAddColScalar(col 10:date, val 10) -> 18:date, VectorUDFDateAddColScalar(col 10:date, val 5) -> 19:date) -> 26:date, IfExprDoubleColumnLongScalar(col 17:boolean, col 28:double, val 0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 27:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 27:double) -> 28:double) -> 27:double, IfExprDoubleColumnDoubleScalar(col 17:boolean, col 29:double, val 0.0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 28:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 28:double) -> 29:double) -> 28:double, IfExprNullColumn(col 17:boolean, null, col 48)(children: StringGroupColEqualStringScalar(col 23:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 17:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 48:decimal(10,2)) -> 30:decimal(10,2), IfExprColumnNull(col 18:boolean, col 49:decimal(10,2), null)(children: StringGroupColEqualStringScalar(col 23:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 18:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 49:decimal(10,2)) -> 31:decimal(10,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 23:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 19:boolean) -> 32:decimal(12,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 23:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 19:boolean) -> 33:decimal(12,2), IfExprDecimal64ScalarDecimal64Column(col 19:boolean, decimal64Val 0, decimalVal 0, col 7:decimal(1,0)/DECIMAL_64)(children: StringGroupColEqualStringScalar(col 23:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 19:boolean) -> 34:decimal(10,2)/DECIMAL_64, IfExprDecimal64ColumnDecimal64Scalar(col 35:boolean, col 7:decimal(10,2)/DECIMAL_64, decimal64Val 0, decimalVal 0)(children: StringGroupColEqualStringScalar(col 23:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 35:boolean) -> 36:decimal(10,2)/DECIMAL_64, IfExprTimestampColumnColumn(col 37:boolean, col 38:timestampcol 39:timestamp)(children: LongColGreaterLongScalar(col 1:int, val 30) -> 37:boolean, CastDateToTimestamp(col 12:date) -> 38:timestamp, CastDateToTimestamp(col 11:date) -> 39:timestamp) -> 40:timestamp, IfExprColumnNull(col 37:boolean, col 41:int, null)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 37:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 41:int) -> 42:int, IfExprNullColumn(col 43:boolean, null, col 44)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 43:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 44:int) -> 45:int, IfExprLongScalarLongScalar(col 47:boolean, val 14245, val 14609)(children: LongColGreaterLongScalar(col 46:int, val 100)(children: LongColModuloLongScalar(col 2:int, val 500) -> 46:int) -> 47:boolean) -> 46:date Statistics: Num rows: 101 Data size: 78500 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -543,7 +543,7 @@ STAGE PLANS: includeColumns: [1, 2, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14] dataColumns: l_orderkey:int, l_partkey:int, l_suppkey:int, l_linenumber:int, l_quantity:int, l_extendedprice:double, l_discount:double, l_tax:decimal(10,2)/DECIMAL_64, l_returnflag:char(1), l_linestatus:char(1), l_shipdate:date, l_commitdate:date, l_receiptdate:date, l_shipinstruct:varchar(20), l_shipmode:char(10), l_comment:string partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, bigint, bigint, bigint, string, string, string, string, string, bigint, double, double, double, decimal(10,2), decimal(10,2), decimal(12,2), decimal(12,2), decimal(10,2), decimal(10,2), timestamp, timestamp, timestamp, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(10,2), decimal(10,2)] + scratchColumnTypeNames: [bigint, bigint, bigint, bigint, string, string, string, string, string, bigint, double, double, double, decimal(10,2), decimal(10,2), decimal(12,2), decimal(12,2), decimal(10,2)/DECIMAL_64, bigint, decimal(10,2)/DECIMAL_64, bigint, timestamp, timestamp, timestamp, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(10,2), decimal(10,2)] Stage: Stage-0 Fetch Operator @@ -844,8 +844,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [4, 27, 38, 48, 52, 54, 60, 63, 65, 67, 68, 69, 70, 73, 76, 79, 80] - selectExpressions: IfExprColumnCondExpr(col 17:boolean, col 18:stringcol 26:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, ConstantVectorExpression(val Single) -> 18:string, IfExprColumnCondExpr(col 19:boolean, col 20:stringcol 25:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 19:boolean, ConstantVectorExpression(val Two) -> 20:string, IfExprColumnCondExpr(col 21:boolean, col 22:stringcol 24:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 21:boolean, ConstantVectorExpression(val Some) -> 22:string, IfExprStringScalarStringScalar(col 23:boolean, val Many, val Huge number)(children: LongColLessLongScalar(col 4:int, val 100) -> 23:boolean) -> 24:string) -> 25:string) -> 26:string) -> 27:string, IfExprColumnCondExpr(col 23:boolean, col 28:stringcol 37:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 23:boolean, ConstantVectorExpression(val Single) -> 28:string, IfExprColumnCondExpr(col 29:boolean, col 30:stringcol 36:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 29:boolean, ConstantVectorExpression(val Two) -> 30:string, IfExprColumnCondExpr(col 31:boolean, col 32:stringcol 35:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 31:boolean, ConstantVectorExpression(val Some) -> 32:string, IfExprColumnNull(col 33:boolean, col 34:string, null)(children: LongColLessLongScalar(col 4:int, val 100) -> 33:boolean, ConstantVectorExpression(val Many) -> 34:string) -> 35:string) -> 36:string) -> 37:string) -> 38:string, IfExprColumnCondExpr(col 39:boolean, col 40:stringcol 47:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 39:boolean, ConstantVectorExpression(val Single) -> 40:string, IfExprColumnCondExpr(col 41:boolean, col 42:stringcol 46:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 41:boolean, ConstantVectorExpression(val Two) -> 42:string, IfExprColumnCondExpr(col 43:boolean, col 44:stringcol 45:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 43:boolean, ConstantVectorExpression(val Some) -> 44:string, IfExprNullNull(null, null) -> 45:string) -> 46:string) -> 47:string) -> 48:string, IfExprCondExprCondExpr(col 49:boolean, col 50:datecol 51:date)(children: StringGroupColEqualCharScalar(col 14:char(10), val SHIP) -> 49:boolean, VectorUDFDateAddColScalar(col 10:date, val 10) -> 50:date, VectorUDFDateAddColScalar(col 10:date, val 5) -> 51:date) -> 52:date, IfExprDoubleColumnLongScalar(col 57:boolean, col 58:double, val 0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 57:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 54:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 54:double) -> 58:double) -> 54:double, IfExprCondExprColumn(col 57:boolean, col 59:double, col 58:double)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 57:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 58:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 58:double) -> 59:double, ConstantVectorExpression(val 0.0) -> 58:double) -> 60:double, IfExprNullColumn(col 62:boolean, null, col 82)(children: StringGroupColEqualStringScalar(col 61:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 62:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 82:decimal(10,2)) -> 63:decimal(10,2), IfExprColumnNull(col 64:boolean, col 83:decimal(10,2), null)(children: StringGroupColEqualStringScalar(col 61:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 64:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 83:decimal(10,2)) -> 65:decimal(10,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 61:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 66:boolean) -> 67:decimal(12,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 61:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 66:boolean) -> 68:decimal(12,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 61:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 66:boolean) -> 69:decimal(10,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 61:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 66:boolean) -> 70:decimal(10,2), IfExprCondExprCondExpr(col 66:boolean, col 71:timestampcol 72:timestamp)(children: LongColGreaterLongScalar(col 1:int, val 30) -> 66:boolean, CastDateToTimestamp(col 12:date) -> 71:timestamp, CastDateToTimestamp(col 11:date) -> 72:timestamp) -> 73:timestamp, IfExprCondExprNull(col 74:boolean, col 75:int, null)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 74:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 75:int) -> 76:int, IfExprNullCondExpr(col 77:boolean, null, col 78:int)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 77:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 78:int) -> 79:int, IfExprLongScalarLongScalar(col 81:boolean, val 14245, val 14609)(children: LongColGreaterLongScalar(col 80:int, val 100)(children: LongColModuloLongScalar(col 2:int, val 500) -> 80:int) -> 81:boolean) -> 80:date + projectedOutputColumnNums: [4, 27, 38, 48, 52, 54, 60, 63, 65, 67, 68, 69, 71, 75, 78, 81, 82] + selectExpressions: IfExprColumnCondExpr(col 17:boolean, col 18:stringcol 26:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, ConstantVectorExpression(val Single) -> 18:string, IfExprColumnCondExpr(col 19:boolean, col 20:stringcol 25:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 19:boolean, ConstantVectorExpression(val Two) -> 20:string, IfExprColumnCondExpr(col 21:boolean, col 22:stringcol 24:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 21:boolean, ConstantVectorExpression(val Some) -> 22:string, IfExprStringScalarStringScalar(col 23:boolean, val Many, val Huge number)(children: LongColLessLongScalar(col 4:int, val 100) -> 23:boolean) -> 24:string) -> 25:string) -> 26:string) -> 27:string, IfExprColumnCondExpr(col 23:boolean, col 28:stringcol 37:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 23:boolean, ConstantVectorExpression(val Single) -> 28:string, IfExprColumnCondExpr(col 29:boolean, col 30:stringcol 36:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 29:boolean, ConstantVectorExpression(val Two) -> 30:string, IfExprColumnCondExpr(col 31:boolean, col 32:stringcol 35:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 31:boolean, ConstantVectorExpression(val Some) -> 32:string, IfExprColumnNull(col 33:boolean, col 34:string, null)(children: LongColLessLongScalar(col 4:int, val 100) -> 33:boolean, ConstantVectorExpression(val Many) -> 34:string) -> 35:string) -> 36:string) -> 37:string) -> 38:string, IfExprColumnCondExpr(col 39:boolean, col 40:stringcol 47:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 39:boolean, ConstantVectorExpression(val Single) -> 40:string, IfExprColumnCondExpr(col 41:boolean, col 42:stringcol 46:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 41:boolean, ConstantVectorExpression(val Two) -> 42:string, IfExprColumnCondExpr(col 43:boolean, col 44:stringcol 45:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 43:boolean, ConstantVectorExpression(val Some) -> 44:string, IfExprNullNull(null, null) -> 45:string) -> 46:string) -> 47:string) -> 48:string, IfExprCondExprCondExpr(col 49:boolean, col 50:datecol 51:date)(children: StringGroupColEqualCharScalar(col 14:char(10), val SHIP) -> 49:boolean, VectorUDFDateAddColScalar(col 10:date, val 10) -> 50:date, VectorUDFDateAddColScalar(col 10:date, val 5) -> 51:date) -> 52:date, IfExprDoubleColumnLongScalar(col 57:boolean, col 58:double, val 0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 57:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 54:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 54:double) -> 58:double) -> 54:double, IfExprCondExprColumn(col 57:boolean, col 59:double, col 58:double)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 57:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 58:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 58:double) -> 59:double, ConstantVectorExpression(val 0.0) -> 58:double) -> 60:double, IfExprNullColumn(col 62:boolean, null, col 84)(children: StringGroupColEqualStringScalar(col 61:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 62:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 84:decimal(10,2)) -> 63:decimal(10,2), IfExprColumnNull(col 64:boolean, col 85:decimal(10,2), null)(children: StringGroupColEqualStringScalar(col 61:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 64:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 85:decimal(10,2)) -> 65:decimal(10,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 61:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 66:boolean) -> 67:decimal(12,2), VectorUDFAdaptor(if((UDFToString(l_shipinstruct) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 61:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 66:boolean) -> 68:decimal(12,2), IfExprDecimal64ScalarDecimal64Column(col 66:boolean, decimal64Val 0, decimalVal 0, col 7:decimal(1,0)/DECIMAL_64)(children: StringGroupColEqualStringScalar(col 61:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 66:boolean) -> 69:decimal(10,2)/DECIMAL_64, IfExprDecimal64ColumnDecimal64Scalar(col 70:boolean, col 7:decimal(10,2)/DECIMAL_64, decimal64Val 0, decimalVal 0)(children: StringGroupColEqualStringScalar(col 61:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 70:boolean) -> 71:decimal(10,2)/DECIMAL_64, IfExprCondExprCondExpr(col 72:boolean, col 73:timestampcol 74:timestamp)(children: LongColGreaterLongScalar(col 1:int, val 30) -> 72:boolean, CastDateToTimestamp(col 12:date) -> 73:timestamp, CastDateToTimestamp(col 11:date) -> 74:timestamp) -> 75:timestamp, IfExprCondExprNull(col 76:boolean, col 77:int, null)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 76:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 77:int) -> 78:int, IfExprNullCondExpr(col 79:boolean, null, col 80:int)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 79:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 80:int) -> 81:int, IfExprLongScalarLongScalar(col 83:boolean, val 14245, val 14609)(children: LongColGreaterLongScalar(col 82:int, val 100)(children: LongColModuloLongScalar(col 2:int, val 500) -> 82:int) -> 83:boolean) -> 82:date Statistics: Num rows: 101 Data size: 78500 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -872,7 +872,7 @@ STAGE PLANS: includeColumns: [1, 2, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14] dataColumns: l_orderkey:int, l_partkey:int, l_suppkey:int, l_linenumber:int, l_quantity:int, l_extendedprice:double, l_discount:double, l_tax:decimal(10,2)/DECIMAL_64, l_returnflag:char(1), l_linestatus:char(1), l_shipdate:date, l_commitdate:date, l_receiptdate:date, l_shipinstruct:varchar(20), l_shipmode:char(10), l_comment:string partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, bigint, bigint, bigint, bigint, double, double, bigint, bigint, double, double, double, string, bigint, decimal(10,2), bigint, decimal(10,2), bigint, decimal(12,2), decimal(12,2), decimal(10,2), decimal(10,2), timestamp, timestamp, timestamp, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(10,2), decimal(10,2)] + scratchColumnTypeNames: [bigint, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, bigint, bigint, bigint, bigint, double, double, bigint, bigint, double, double, double, string, bigint, decimal(10,2), bigint, decimal(10,2), bigint, decimal(12,2), decimal(12,2), decimal(10,2)/DECIMAL_64, bigint, decimal(10,2)/DECIMAL_64, bigint, timestamp, timestamp, timestamp, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(10,2), decimal(10,2)] Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/vector_decimal_mapjoin.q.out ql/src/test/results/clientpositive/vector_decimal_mapjoin.q.out index 0f090bf..aa9f54b 100644 --- ql/src/test/results/clientpositive/vector_decimal_mapjoin.q.out +++ ql/src/test/results/clientpositive/vector_decimal_mapjoin.q.out @@ -690,7 +690,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(14,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(14,2)/DECIMAL_64) -> 3:decimal(14,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(14,2)/DECIMAL_64) predicate: dec is not null (type: boolean) Statistics: Num rows: 1049 Data size: 11234 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -708,8 +708,8 @@ STAGE PLANS: 0 _col0 (type: decimal(16,2)) 1 _col0 (type: decimal(16,2)) Map Join Vectorization: - bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 4:decimal(16,2) - bigTableValueExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 5:decimal(16,2) + bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 3:decimal(16,2) + bigTableValueExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 4:decimal(16,2) className: VectorMapJoinOperator native: false nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true @@ -742,7 +742,7 @@ STAGE PLANS: includeColumns: [0] dataColumns: dec:decimal(14,2)/DECIMAL_64, value_dec:decimal(14,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(14,2), decimal(16,2), decimal(16,2)] + scratchColumnTypeNames: [decimal(16,2), decimal(16,2)] Local Work: Map Reduce Local Work @@ -933,7 +933,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: SelectColumnIsNotNull(col 3:decimal(14,2))(children: ConvertDecimal64ToDecimal(col 0:decimal(14,2)/DECIMAL_64) -> 3:decimal(14,2)) + predicateExpression: SelectColumnIsNotNull(col 0:decimal(14,2)/DECIMAL_64) predicate: dec is not null (type: boolean) Statistics: Num rows: 1049 Data size: 11234 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -951,8 +951,8 @@ STAGE PLANS: 0 _col0 (type: decimal(16,2)) 1 _col0 (type: decimal(16,2)) Map Join Vectorization: - bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 4:decimal(16,2) - bigTableValueExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 5:decimal(16,2), ConvertDecimal64ToDecimal(col 1:decimal(14,2)/DECIMAL_64) -> 3:decimal(14,2) + bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 3:decimal(16,2) + bigTableValueExpressions: ConvertDecimal64ToDecimal(col 0:decimal(16,2)/DECIMAL_64) -> 4:decimal(16,2), ConvertDecimal64ToDecimal(col 1:decimal(14,2)/DECIMAL_64) -> 5:decimal(14,2) className: VectorMapJoinOperator native: false nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true @@ -985,7 +985,7 @@ STAGE PLANS: includeColumns: [0, 1] dataColumns: dec:decimal(14,2)/DECIMAL_64, value_dec:decimal(14,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(14,2), decimal(16,2), decimal(16,2), decimal(14,0)] + scratchColumnTypeNames: [decimal(16,2), decimal(16,2), decimal(14,2), decimal(14,0)] Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/vectorized_case.q.out ql/src/test/results/clientpositive/vectorized_case.q.out index 625162e..828131f 100644 --- ql/src/test/results/clientpositive/vectorized_case.q.out +++ ql/src/test/results/clientpositive/vectorized_case.q.out @@ -527,7 +527,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [6] - selectExpressions: IfExprCondExprCondExpr(col 3:boolean, col 7:decimal(11,0)col 8:decimal(11,0))(children: VectorUDFAdaptor((member = 1)) -> 3:boolean, ConvertDecimal64ToDecimal(col 4:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 4:decimal(11,0)/DECIMAL_64) -> 7:decimal(11,0), ConvertDecimal64ToDecimal(col 5:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 2, decimalVal 2) -> 5:decimal(11,0)/DECIMAL_64) -> 8:decimal(11,0)) -> 6:decimal(11,0) + selectExpressions: IfExprCondExprCondExpr(col 3:boolean, col 7:decimal(11,0)col 8:decimal(11,0))(children: Decimal64ColEqualDecimal64Scalar(col 0:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 3:boolean, ConvertDecimal64ToDecimal(col 4:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 4:decimal(11,0)/DECIMAL_64) -> 7:decimal(11,0), ConvertDecimal64ToDecimal(col 5:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 2, decimalVal 2) -> 5:decimal(11,0)/DECIMAL_64) -> 8:decimal(11,0)) -> 6:decimal(11,0) Statistics: Num rows: 3 Data size: 672 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -547,7 +547,7 @@ STAGE PLANS: featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 2 @@ -604,7 +604,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [8] - selectExpressions: VectorUDFAdaptor(CASE WHEN ((member = 1)) THEN (1) ELSE ((attr + 2)) END)(children: VectorUDFAdaptor((member = 1)) -> 6:boolean, ConvertDecimal64ToDecimal(col 7:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 2, decimalVal 2) -> 7:decimal(11,0)/DECIMAL_64) -> 9:decimal(11,0)) -> 8:decimal(11,0) + selectExpressions: IfExprDecimal64ScalarDecimal64Column(col 6:boolean, decimal64Val 1, decimalVal 1, col 7:decimal(1,0)/DECIMAL_64)(children: Decimal64ColEqualDecimal64Scalar(col 0:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 6:boolean, Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 2, decimalVal 2) -> 7:decimal(11,0)/DECIMAL_64) -> 8:decimal(11,0)/DECIMAL_64 Statistics: Num rows: 3 Data size: 672 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -624,14 +624,14 @@ STAGE PLANS: featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 2 includeColumns: [0, 1] dataColumns: member:decimal(10,0)/DECIMAL_64, attr:decimal(10,0)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, decimal(1,0), decimal(11,0)/DECIMAL_64, bigint, decimal(11,0)/DECIMAL_64, decimal(11,0), decimal(11,0)] + scratchColumnTypeNames: [bigint, decimal(1,0), decimal(11,0)/DECIMAL_64, bigint, decimal(11,0)/DECIMAL_64, decimal(11,0)/DECIMAL_64] Stage: Stage-0 Fetch Operator @@ -681,7 +681,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [8] - selectExpressions: VectorUDFAdaptor(CASE WHEN ((member = 1)) THEN ((attr + 1)) ELSE (2) END)(children: VectorUDFAdaptor((member = 1)) -> 6:boolean, ConvertDecimal64ToDecimal(col 7:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 7:decimal(11,0)/DECIMAL_64) -> 9:decimal(11,0)) -> 8:decimal(11,0) + selectExpressions: IfExprDecimal64ColumnDecimal64Scalar(col 6:boolean, col 7:decimal(11,0)/DECIMAL_64, decimal64Val 2, decimalVal 2)(children: Decimal64ColEqualDecimal64Scalar(col 0:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 6:boolean, Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 7:decimal(11,0)/DECIMAL_64) -> 8:decimal(11,0)/DECIMAL_64 Statistics: Num rows: 3 Data size: 672 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -701,14 +701,14 @@ STAGE PLANS: featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 2 includeColumns: [0, 1] dataColumns: member:decimal(10,0)/DECIMAL_64, attr:decimal(10,0)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, decimal(11,0)/DECIMAL_64, decimal(1,0), bigint, decimal(11,0)/DECIMAL_64, decimal(11,0), decimal(11,0)] + scratchColumnTypeNames: [bigint, decimal(11,0)/DECIMAL_64, decimal(1,0), bigint, decimal(11,0)/DECIMAL_64, decimal(11,0)/DECIMAL_64] Stage: Stage-0 Fetch Operator diff --git storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java index c1d6a3a..e074fb9 100644 --- storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java +++ storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java @@ -62,8 +62,27 @@ public void fill(HiveDecimal value) { } @Override + // Simplify vector by brute-force flattening noNulls and isRepeating + // This can be used to reduce combinatorial explosion of code paths in VectorExpressions + // with many arguments. public void flatten(boolean selectedInUse, int[] sel, int size) { - throw new RuntimeException("Not implemented"); + flattenPush(); + if (isRepeating) { + isRepeating = false; + HiveDecimalWritable repeat = vector[0]; + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + vector[i].set(repeat); + } + } else { + for (int i = 0; i < size; i++) { + vector[i].set(repeat); + } + } + flattenRepeatingNulls(selectedInUse, sel, size); + } + flattenNoNulls(selectedInUse, sel, size); } /** diff --git storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringExpr.java storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringExpr.java index 162e8e6..bcbad4b 100644 --- storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringExpr.java +++ storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringExpr.java @@ -129,6 +129,23 @@ public static int characterCount(byte[] bytes, int start, int length) { return charCount; } + public static void padRight(BytesColumnVector outV, int i, byte[] bytes, int start, int length, + int maxCharacterLength) { + + final int characterLength = StringExpr.characterCount(bytes, start, length); + final int blankPadLength = Math.max(maxCharacterLength - characterLength, 0); + final int resultLength = length + blankPadLength; + outV.ensureValPreallocated(resultLength); + byte[] resultBytes = outV.getValPreallocatedBytes(); + final int resultStart = outV.getValPreallocatedStart(); + System.arraycopy(bytes, start, resultBytes, resultStart, length); + final int padEnd = resultStart + resultLength; + for (int p = resultStart + length; p < padEnd; p++) { + resultBytes[p] = ' '; + } + outV.setValPreallocated(i, resultLength); + } + // A setVal with the same function signature as rightTrim, leftTrim, truncate, etc, below. // Useful for class generation via templates. public static void assign(BytesColumnVector outV, int i, byte[] bytes, int start, int length) { diff --git vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java index 8b36371..666572a 100644 --- vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java +++ vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java @@ -316,16 +316,22 @@ {"Decimal64ColumnArithmeticDecimal64Column", "Add", "+"}, {"Decimal64ColumnArithmeticDecimal64Column", "Subtract", "-"}, + {"ColumnCompareScalar", "Equal", "long", "long", "=="}, {"ColumnCompareScalar", "Equal", "long", "double", "=="}, {"ColumnCompareScalar", "Equal", "double", "double", "=="}, + {"ColumnCompareScalar", "NotEqual", "long", "long", "!="}, {"ColumnCompareScalar", "NotEqual", "long", "double", "!="}, {"ColumnCompareScalar", "NotEqual", "double", "double", "!="}, + {"ColumnCompareScalar", "Less", "long", "long", "<"}, {"ColumnCompareScalar", "Less", "long", "double", "<"}, {"ColumnCompareScalar", "Less", "double", "double", "<"}, + {"ColumnCompareScalar", "LessEqual", "long", "long", "<="}, {"ColumnCompareScalar", "LessEqual", "long", "double", "<="}, {"ColumnCompareScalar", "LessEqual", "double", "double", "<="}, + {"ColumnCompareScalar", "Greater", "long", "long", ">"}, {"ColumnCompareScalar", "Greater", "long", "double", ">"}, {"ColumnCompareScalar", "Greater", "double", "double", ">"}, + {"ColumnCompareScalar", "GreaterEqual", "long", "long", ">="}, {"ColumnCompareScalar", "GreaterEqual", "long", "double", ">="}, {"ColumnCompareScalar", "GreaterEqual", "double", "double", ">="}, @@ -336,16 +342,22 @@ {"ColumnCompareScalar", "Greater", "double", "long", ">"}, {"ColumnCompareScalar", "GreaterEqual", "double", "long", ">="}, + {"ScalarCompareColumn", "Equal", "long", "long", "=="}, {"ScalarCompareColumn", "Equal", "long", "double", "=="}, {"ScalarCompareColumn", "Equal", "double", "double", "=="}, + {"ScalarCompareColumn", "NotEqual", "long", "long", "!="}, {"ScalarCompareColumn", "NotEqual", "long", "double", "!="}, {"ScalarCompareColumn", "NotEqual", "double", "double", "!="}, + {"ScalarCompareColumn", "Less", "long", "long", "<"}, {"ScalarCompareColumn", "Less", "long", "double", "<"}, {"ScalarCompareColumn", "Less", "double", "double", "<"}, + {"ScalarCompareColumn", "LessEqual", "long", "long", "<="}, {"ScalarCompareColumn", "LessEqual", "long", "double", "<="}, {"ScalarCompareColumn", "LessEqual", "double", "double", "<="}, + {"ScalarCompareColumn", "Greater", "long", "long", ">"}, {"ScalarCompareColumn", "Greater", "long", "double", ">"}, {"ScalarCompareColumn", "Greater", "double", "double", ">"}, + {"ScalarCompareColumn", "GreaterEqual", "long", "long", ">="}, {"ScalarCompareColumn", "GreaterEqual", "long", "double", ">="}, {"ScalarCompareColumn", "GreaterEqual", "double", "double", ">="}, @@ -356,6 +368,28 @@ {"ScalarCompareColumn", "Greater", "double", "long", ">"}, {"ScalarCompareColumn", "GreaterEqual", "double", "long", ">="}, + // Compare decimal to decimal. + {"DecimalCompareDecimal", "Equal", "==", "Col", "Column"}, + {"DecimalCompareDecimal", "NotEqual", "!=", "Col", "Column"}, + {"DecimalCompareDecimal", "Less", "<", "Col", "Column"}, + {"DecimalCompareDecimal", "LessEqual", "<=", "Col", "Column"}, + {"DecimalCompareDecimal", "Greater", ">", "Col", "Column"}, + {"DecimalCompareDecimal", "GreaterEqual", ">=", "Col", "Column"}, + + {"DecimalCompareDecimal", "Equal", "==", "Col", "Scalar"}, + {"DecimalCompareDecimal", "NotEqual", "!=", "Col", "Scalar"}, + {"DecimalCompareDecimal", "Less", "<", "Col", "Scalar"}, + {"DecimalCompareDecimal", "LessEqual", "<=", "Col", "Scalar"}, + {"DecimalCompareDecimal", "Greater", ">", "Col", "Scalar"}, + {"DecimalCompareDecimal", "GreaterEqual", ">=", "Col", "Scalar"}, + + {"DecimalCompareDecimal", "Equal", "==", "Scalar", "Column"}, + {"DecimalCompareDecimal", "NotEqual", "!=", "Scalar", "Column"}, + {"DecimalCompareDecimal", "Less", "<", "Scalar", "Column"}, + {"DecimalCompareDecimal", "LessEqual", "<=", "Scalar", "Column"}, + {"DecimalCompareDecimal", "Greater", ">", "Scalar", "Column"}, + {"DecimalCompareDecimal", "GreaterEqual", ">=", "Scalar", "Column"}, + // Compare timestamp to timestamp. {"TimestampCompareTimestamp", "Equal", "==", "timestamp", "Col", "Column"}, {"TimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Col", "Column"}, @@ -478,6 +512,28 @@ {"LongDoubleCompareTimestamp", "GreaterEqual", "long", ">=", "Scalar", "Column"}, {"LongDoubleCompareTimestamp", "GreaterEqual", "double", ">=", "Scalar", "Column"}, + // Decimal64 + {"Decimal64ColumnCompareDecimal64Scalar", "Equal"}, + {"Decimal64ColumnCompareDecimal64Scalar", "NotEqual"}, + {"Decimal64ColumnCompareDecimal64Scalar", "Less"}, + {"Decimal64ColumnCompareDecimal64Scalar", "LessEqual"}, + {"Decimal64ColumnCompareDecimal64Scalar", "Greater"}, + {"Decimal64ColumnCompareDecimal64Scalar", "GreaterEqual"}, + + {"Decimal64ScalarCompareDecimal64Column", "Equal"}, + {"Decimal64ScalarCompareDecimal64Column", "NotEqual"}, + {"Decimal64ScalarCompareDecimal64Column", "Less"}, + {"Decimal64ScalarCompareDecimal64Column", "LessEqual"}, + {"Decimal64ScalarCompareDecimal64Column", "Greater"}, + {"Decimal64ScalarCompareDecimal64Column", "GreaterEqual"}, + + {"Decimal64ColumnCompareDecimal64Column", "Equal"}, + {"Decimal64ColumnCompareDecimal64Column", "NotEqual"}, + {"Decimal64ColumnCompareDecimal64Column", "Less"}, + {"Decimal64ColumnCompareDecimal64Column", "LessEqual"}, + {"Decimal64ColumnCompareDecimal64Column", "Greater"}, + {"Decimal64ColumnCompareDecimal64Column", "GreaterEqual"}, + // Filter long/double. {"FilterColumnCompareScalar", "Equal", "long", "double", "=="}, {"FilterColumnCompareScalar", "Equal", "double", "double", "=="}, @@ -884,16 +940,22 @@ {"FilterColumnBetweenDynamicValue", "date", ""}, {"FilterColumnBetweenDynamicValue", "timestamp", ""}, + {"ColumnCompareColumn", "Equal", "long", "long", "=="}, {"ColumnCompareColumn", "Equal", "long", "double", "=="}, {"ColumnCompareColumn", "Equal", "double", "double", "=="}, + {"ColumnCompareColumn", "NotEqual", "long", "long", "!="}, {"ColumnCompareColumn", "NotEqual", "long", "double", "!="}, {"ColumnCompareColumn", "NotEqual", "double", "double", "!="}, + {"ColumnCompareColumn", "Less", "long", "long", "<"}, {"ColumnCompareColumn", "Less", "long", "double", "<"}, {"ColumnCompareColumn", "Less", "double", "double", "<"}, + {"ColumnCompareColumn", "LessEqual", "long", "long", "<="}, {"ColumnCompareColumn", "LessEqual", "long", "double", "<="}, {"ColumnCompareColumn", "LessEqual", "double", "double", "<="}, + {"ColumnCompareColumn", "Greater", "long", "long", ">"}, {"ColumnCompareColumn", "Greater", "long", "double", ">"}, {"ColumnCompareColumn", "Greater", "double", "double", ">"}, + {"ColumnCompareColumn", "GreaterEqual", "long", "long", ">="}, {"ColumnCompareColumn", "GreaterEqual", "long", "double", ">="}, {"ColumnCompareColumn", "GreaterEqual", "double", "double", ">="}, @@ -1063,6 +1125,11 @@ {"IfExprScalarScalar", "long", "double"}, {"IfExprScalarScalar", "double", "double"}, + {"IfExprObjectColumnColumn", "decimal"}, + {"IfExprObjectColumnScalar", "decimal"}, + {"IfExprObjectScalarColumn", "decimal"}, + {"IfExprObjectScalarScalar", "decimal"}, + {"IfExprObjectColumnColumn", "timestamp"}, {"IfExprObjectColumnColumn", "interval_day_time"}, {"IfExprObjectColumnScalar", "timestamp"}, @@ -1271,6 +1338,9 @@ private void generate() throws Exception { } else if (tdesc[0].equals("ScalarCompareColumn")) { generateScalarCompareColumn(tdesc); + } else if (tdesc[0].equals("DecimalCompareDecimal")) { + generateDecimalCompareDecimal(tdesc); + } else if (tdesc[0].equals("TimestampCompareTimestamp")) { generateTimestampCompareTimestamp(tdesc); @@ -1388,6 +1458,12 @@ private void generate() throws Exception { generateFilterStringGroupColumnCompareStringGroupColumn(tdesc); } else if (tdesc[0].equals("StringGroupColumnCompareStringGroupColumn")) { generateStringGroupColumnCompareStringGroupColumn(tdesc); + } else if (tdesc[0].equals("Decimal64ColumnCompareDecimal64Scalar")) { + generateDecimal64ColumnCompareDecimal64Scalar(tdesc); + } else if (tdesc[0].equals("Decimal64ScalarCompareDecimal64Column")) { + generateDecimal64ScalarCompareDecimal64Column(tdesc); + } else if (tdesc[0].equals("Decimal64ColumnCompareDecimal64Column")) { + generateDecimal64ColumnCompareDecimal64Column(tdesc); } else if (tdesc[0].equals("IfExprColumnScalar")) { generateIfExprColumnScalar(tdesc); } else if (tdesc[0].equals("IfExprScalarColumn")) { @@ -2253,6 +2329,7 @@ private void generateIfExprObject(String [] tdesc) throws Exception { String objectName; String scalarType; String scalarImport; + String ifDefined = ""; if (typeName.equals("timestamp")) { objectName = "Timestamp"; scalarType = "Timestamp"; @@ -2261,6 +2338,11 @@ private void generateIfExprObject(String [] tdesc) throws Exception { objectName = "IntervalDayTime"; scalarType = "HiveIntervalDayTime"; scalarImport = "org.apache.hadoop.hive.common.type.HiveIntervalDayTime"; + } else if (typeName.equals("decimal")) { + ifDefined = "DECIMAL"; + objectName = "Decimal"; + scalarType = "HiveDecimal"; + scalarImport = "org.apache.hadoop.hive.common.type.HiveDecimal"; } else { objectName = "unknown"; scalarType = "unknown"; @@ -2276,6 +2358,8 @@ private void generateIfExprObject(String [] tdesc) throws Exception { File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); String templateString = readFile(templateFile); + templateString = evaluateIfDefined(templateString, ifDefined); + templateString = templateString.replaceAll("", className); templateString = templateString.replaceAll("", scalarType); templateString = templateString.replaceAll("", scalarImport); @@ -2472,6 +2556,23 @@ private void generateColumnCompareOperatorColumn(String[] tdesc, boolean filter, } } + private void generateDecimalCompareDecimal(String[] tdesc) throws Exception { + String operatorName = tdesc[1]; + String operatorSymbol = tdesc[2]; + String className = "Decimal" + tdesc[3] + operatorName + "Decimal" + tdesc[4]; + + //Read the template into a string; + String fileName = "Decimal" + (tdesc[3].equals("Col") ? "Column" : tdesc[3]) + "CompareDecimal" + + (tdesc[4].equals("Col") ? "Column" : tdesc[4]); + File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", operatorSymbol); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + // ----------------------------------------------------------------------------------------------- // // Filter timestamp against timestamp, long (seconds), and double (seconds with fractional @@ -3038,6 +3139,27 @@ private void generateColumnDivideColumnDecimal(String[] tdesc) throws IOExceptio className, templateString); } + private void generateDecimal64ColumnCompareDecimal64Scalar(String[] tdesc) throws IOException { + String operatorName = tdesc[1]; + String className = "Decimal64Col" + operatorName + "Decimal64Scalar"; + String baseClassName = "LongCol" + operatorName + "LongScalar"; + generateDecimal64ColumnCompare(tdesc, className, baseClassName); + } + + private void generateDecimal64ScalarCompareDecimal64Column(String[] tdesc) throws IOException { + String operatorName = tdesc[1]; + String className = "Decimal64Scalar" + operatorName + "Decimal64Column"; + String baseClassName = "LongScalar" + operatorName + "LongColumn"; + generateDecimal64ColumnCompare(tdesc, className, baseClassName); + } + + private void generateDecimal64ColumnCompareDecimal64Column(String[] tdesc) throws IOException { + String operatorName = tdesc[1]; + String className = "Decimal64Col" + operatorName + "Decimal64Column"; + String baseClassName = "LongCol" + operatorName + "LongColumn"; + generateDecimal64ColumnCompare(tdesc, className, baseClassName); + } + private void generateScalarArithmeticColumn(String[] tdesc) throws Exception { String operatorName = tdesc[1]; String operandType1 = tdesc[2]; @@ -3124,7 +3246,7 @@ private void generateDTIScalarCompareColumn(String[] tdesc) throws Exception { String operandType = tdesc[2]; String className = getCamelCaseType(operandType) + "Scalar" + operatorName + getCamelCaseType(operandType) + "Column"; - String baseClassName = "org.apache.hadoop.hive.ql.exec.vector.expressions.LongScalar" + operatorName + "LongColumn"; + String baseClassName = "org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongScalar" + operatorName + "LongColumn"; //Read the template into a string; File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); String templateString = readFile(templateFile); @@ -3183,7 +3305,7 @@ private void generateDTIColumnCompareScalar(String[] tdesc) throws Exception { String operandType = tdesc[2]; String className = getCamelCaseType(operandType) + "Col" + operatorName + getCamelCaseType(operandType) + "Scalar"; - String baseClassName = "org.apache.hadoop.hive.ql.exec.vector.expressions.LongCol" + operatorName + "LongScalar"; + String baseClassName = "org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongCol" + operatorName + "LongScalar"; //Read the template into a string; File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); String templateString = readFile(templateFile);