diff --git ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java index efccba0..8c2c1fa 100644 --- ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java +++ ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java @@ -241,7 +241,16 @@ {"FilterStringColumnBetween", ""}, {"FilterStringColumnBetween", "!"}, - + + {"FilterStringColumnBetweenColumnAndScalar", ""}, + {"FilterStringColumnBetweenColumnAndScalar", "!"}, + + {"FilterStringColumnBetweenScalarAndColumn", ""}, + {"FilterStringColumnBetweenScalarAndColumn", "!"}, + + {"FilterStringColumnBetweenColumnAndColumn", ""}, + {"FilterStringColumnBetweenColumnAndColumn", "!"}, + {"StringColumnCompareScalar", "Equal", "=="}, {"StringColumnCompareScalar", "NotEqual", "!="}, {"StringColumnCompareScalar", "Less", "<"}, @@ -328,10 +337,30 @@ {"FilterColumnBetween", "double", ""}, {"FilterColumnBetween", "long", "!"}, {"FilterColumnBetween", "double", "!"}, - - {"FilterDecimalColumnBetween", ""}, - {"FilterDecimalColumnBetween", "!"}, - + + {"FilterColumnBetweenScalarAndColumn", "long", ""}, + {"FilterColumnBetweenScalarAndColumn", "double", ""}, + {"FilterColumnBetweenScalarAndColumn", "long", "!"}, + {"FilterColumnBetweenScalarAndColumn", "double", "!"}, + {"FilterColumnBetweenColumnAndScalar", "long", ""}, + {"FilterColumnBetweenColumnAndScalar", "double", ""}, + {"FilterColumnBetweenColumnAndScalar", "long", "!"}, + {"FilterColumnBetweenColumnAndScalar", "double", "!"}, + {"FilterColumnBetweenColumnAndColumn", "long", ""}, + {"FilterColumnBetweenColumnAndColumn", "double", ""}, + {"FilterColumnBetweenColumnAndColumn", "long", "!"}, + {"FilterColumnBetweenColumnAndColumn", "double", "!"}, + + + {"FilterDecimalColumnBetween", ""}, + {"FilterDecimalColumnBetween", "!"}, + {"FilterDecimalColumnBetweenScalarAndColumn", ""}, + {"FilterDecimalColumnBetweenScalarAndColumn", "!"}, + {"FilterDecimalColumnBetweenColumnAndScalar", ""}, + {"FilterDecimalColumnBetweenColumnAndScalar", "!"}, + {"FilterDecimalColumnBetweenColumnAndColumn", ""}, + {"FilterDecimalColumnBetweenColumnAndColumn", "!"}, + {"ColumnCompareColumn", "Equal", "long", "double", "=="}, {"ColumnCompareColumn", "Equal", "double", "double", "=="}, {"ColumnCompareColumn", "NotEqual", "long", "double", "!="}, @@ -630,6 +659,12 @@ private void generate() throws Exception { generateFilterScalarCompareColumn(tdesc); } else if (tdesc[0].equals("FilterColumnBetween")) { generateFilterColumnBetween(tdesc); + } else if (tdesc[0].equals("FilterColumnBetweenColumnAndColumn")) { + generateFilterColumnBetweenColumnAndColumn(tdesc); + } else if (tdesc[0].equals("FilterColumnBetweenColumnAndScalar")) { + generateFilterColumnBetweenColumnAndScalar(tdesc); + } else if (tdesc[0].equals("FilterColumnBetweenScalarAndColumn")) { + generateFilterColumnBetweenScalarAndColumn(tdesc); } else if (tdesc[0].equals("ScalarArithmeticColumn") || tdesc[0].equals("ScalarDivideColumn")) { generateScalarArithmeticColumn(tdesc); } else if (tdesc[0].equals("FilterColumnCompareColumn")) { @@ -662,8 +697,20 @@ private void generate() throws Exception { generateFilterStringColumnCompareScalar(tdesc); } else if (tdesc[0].equals("FilterStringColumnBetween")) { generateFilterStringColumnBetween(tdesc); + } else if (tdesc[0].equals("FilterStringColumnBetweenColumnAndColumn")) { + generateFilterStringColumnBetweenColumnAndColumn(tdesc); + } else if (tdesc[0].equals("FilterStringColumnBetweenColumnAndScalar")) { + generateFilterStringColumnBetweenColumnAndScalar(tdesc); + } else if (tdesc[0].equals("FilterStringColumnBetweenScalarAndColumn")) { + generateFilterStringColumnBetweenScalarAndColumn(tdesc); + } else if (tdesc[0].equals("FilterDecimalColumnBetweenColumnAndColumn")) { + generateFilterDecimalColumnBetweenColumnAndColumn(tdesc); + } else if (tdesc[0].equals("FilterDecimalColumnBetweenColumnAndScalar")) { + generateFilterDecimalColumnBetweenColumnAndScalar(tdesc); + } else if (tdesc[0].equals("FilterDecimalColumnBetweenScalarAndColumn")) { + generateFilterDecimalColumnBetweenScalarAndColumn(tdesc); } else if (tdesc[0].equals("FilterDecimalColumnBetween")) { - generateFilterDecimalColumnBetween(tdesc); + generateFilterDecimalColumnBetween(tdesc); } else if (tdesc[0].equals("StringColumnCompareScalar")) { generateStringColumnCompareScalar(tdesc); } else if (tdesc[0].equals("FilterStringScalarCompareColumn")) { @@ -710,6 +757,48 @@ private void generateFilterStringColumnBetween(String[] tdesc) throws IOExceptio className, templateString); } + private void generateFilterStringColumnBetweenColumnAndColumn(String[] tdesc) throws IOException { + String optionalNot = tdesc[1]; + String className = "FilterStringColumn" + (optionalNot.equals("!") ? "Not" : "") + + "BetweenColumnAndColumn"; + // Read the template into a string, expand it, and write it. + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", optionalNot); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + + private void generateFilterStringColumnBetweenColumnAndScalar(String[] tdesc) throws IOException { + String optionalNot = tdesc[1]; + String className = "FilterStringColumn" + (optionalNot.equals("!") ? "Not" : "") + + "BetweenColumnAndScalar"; + // Read the template into a string, expand it, and write it. + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", optionalNot); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + + private void generateFilterStringColumnBetweenScalarAndColumn(String[] tdesc) throws IOException { + String optionalNot = tdesc[1]; + String className = "FilterStringColumn" + (optionalNot.equals("!") ? "Not" : "") + + "BetweenScalarAndColumn"; + // Read the template into a string, expand it, and write it. + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", optionalNot); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + private void generateFilterDecimalColumnBetween(String[] tdesc) throws IOException { String optionalNot = tdesc[1]; String className = "FilterDecimalColumn" + (optionalNot.equals("!") ? "Not" : "") @@ -723,7 +812,49 @@ private void generateFilterDecimalColumnBetween(String[] tdesc) throws IOExcepti writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, className, templateString); } + + private void generateFilterDecimalColumnBetweenColumnAndColumn(String[] tdesc) throws IOException { + String optionalNot = tdesc[1]; + String className = "FilterDecimalColumn" + (optionalNot.equals("!") ? "Not" : "") + + "BetweenColumnAndColumn"; + // Read the template into a string, expand it, and write it. + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", optionalNot); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + private void generateFilterDecimalColumnBetweenColumnAndScalar(String[] tdesc) throws IOException { + String optionalNot = tdesc[1]; + String className = "FilterDecimalColumn" + (optionalNot.equals("!") ? "Not" : "") + + "BetweenColumnAndScalar"; + // Read the template into a string, expand it, and write it. + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", optionalNot); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + + private void generateFilterDecimalColumnBetweenScalarAndColumn(String[] tdesc) throws IOException { + String optionalNot = tdesc[1]; + String className = "FilterDecimalColumn" + (optionalNot.equals("!") ? "Not" : "") + + "BetweenScalarAndColumn"; + // Read the template into a string, expand it, and write it. + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", optionalNot); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + private void generateFilterColumnBetween(String[] tdesc) throws Exception { String operandType = tdesc[1]; String optionalNot = tdesc[2]; @@ -743,7 +874,67 @@ private void generateFilterColumnBetween(String[] tdesc) throws Exception { writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, className, templateString); } - + + private void generateFilterColumnBetweenColumnAndColumn(String[] tdesc) throws Exception { + String operandType = tdesc[1]; + String optionalNot = tdesc[2]; + + String className = "Filter" + getCamelCaseType(operandType) + "Column" + + (optionalNot.equals("!") ? "Not" : "") + "BetweenColumnAndColumn"; + String inputColumnVectorType = getColumnVectorType(operandType); + + // Read the template into a string, expand it, and write it. + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", inputColumnVectorType); + templateString = templateString.replaceAll("", operandType); + templateString = templateString.replaceAll("", optionalNot); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + + private void generateFilterColumnBetweenColumnAndScalar(String[] tdesc) throws Exception { + String operandType = tdesc[1]; + String optionalNot = tdesc[2]; + + String className = "Filter" + getCamelCaseType(operandType) + "Column" + + (optionalNot.equals("!") ? "Not" : "") + "BetweenColumnAndScalar"; + String inputColumnVectorType = getColumnVectorType(operandType); + + // Read the template into a string, expand it, and write it. + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", inputColumnVectorType); + templateString = templateString.replaceAll("", operandType); + templateString = templateString.replaceAll("", optionalNot); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + + private void generateFilterColumnBetweenScalarAndColumn(String[] tdesc) throws Exception { + String operandType = tdesc[1]; + String optionalNot = tdesc[2]; + + String className = "Filter" + getCamelCaseType(operandType) + "Column" + + (optionalNot.equals("!") ? "Not" : "") + "BetweenScalarAndColumn"; + String inputColumnVectorType = getColumnVectorType(operandType); + + // Read the template into a string, expand it, and write it. + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", inputColumnVectorType); + templateString = templateString.replaceAll("", operandType); + templateString = templateString.replaceAll("", optionalNot); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + private void generateColumnCompareColumn(String[] tdesc) throws Exception { //The variables are all same as ColumnCompareScalar except that //this template doesn't need a return type. Pass anything as return type. diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterColumnBetweenColumnAndColumn.txt ql/src/gen/vectorization/ExpressionTemplates/FilterColumnBetweenColumnAndColumn.txt new file mode 100644 index 0000000..56ff01d --- /dev/null +++ ql/src/gen/vectorization/ExpressionTemplates/FilterColumnBetweenColumnAndColumn.txt @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + + +/** + * Generated from template FilterColumnBetween.txt, which covers [NOT] BETWEEN filter + * expressions where a column is [NOT] between one scalar and another. + * Output is not produced in a separate column. The selected vector of the input + * {@link VectorizedRowBatch} is updated for in-place filtering. + */ +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int colNum; + private int colNum1; + private int colNum2; + + public (int colNum, int colNum1, int colNum2) { + this.colNum = colNum; + this.colNum1 = colNum1; + this.colNum2 = colNum2; + } + + public () { + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + if (childExpressions != null) { + super.evaluateChildren(batch); + } + inputColVector = () batch.cols[colNum]; + inputColVector1 = () batch.cols[colNum1]; + inputColVector2 = () batch.cols[colNum2]; + int[] sel = batch.selected; + int n = batch.size; + [] vector = inputColVector.vector; + [] vector1 = inputColVector1.vector; + [] vector2 = inputColVector2.vector; + boolean[] nullPos = inputColVector.isNull; + boolean[] nullPos1 = inputColVector1.isNull; + boolean[] nullPos2 = inputColVector2.isNull; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + int newSize = 0; + for (int j = 0; j != n; j++) { + int i = batch.selectedInUse ? sel[j] : j; + vi = inputColVector.isRepeating ? vector[0] : vector[i]; + v1i = inputColVector1.isRepeating ? vector1[0] : vector1[i]; + v2i = inputColVector2.isRepeating ? vector2[0] : vector2[i]; + boolean ni = inputColVector.noNulls || (!inputColVector.noNulls && inputColVector.isRepeating && !nullPos[0]) || + (!inputColVector.noNulls && !inputColVector.isRepeating && !nullPos[i]); + boolean n1i = inputColVector1.noNulls || (!inputColVector1.noNulls && inputColVector1.isRepeating && !nullPos1[0]) || + (!inputColVector1.noNulls && !inputColVector1.isRepeating && !nullPos1[i]); + boolean n2i = inputColVector2.noNulls || (!inputColVector2.noNulls && inputColVector2.isRepeating && !nullPos2[0]) || + (!inputColVector2.noNulls && !inputColVector2.isRepeating && !nullPos2[i]); + + if (ni && n1i && n2i && (vi >= v1i && vi <= v2i)) { + sel[newSize++] = i; + } + } + + if (batch.selectedInUse) { + batch.size = newSize; + } else if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + + @Override + public int getOutputColumn() { + return -1; + } + + @Override + public String getOutputType() { + return "boolean"; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + public int getColNum1() { + return colNum1; + } + + public void setColNum1(int colNum1) { + this.colNum1 = colNum1; + } + + public int getColNum2() { + return colNum2; + } + + public void setColNum2(int colNum2) { + this.colNum2 = colNum2; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.FILTER) + .setNumArguments(3) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string")) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } + +} diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterColumnBetweenColumnAndScalar.txt ql/src/gen/vectorization/ExpressionTemplates/FilterColumnBetweenColumnAndScalar.txt new file mode 100644 index 0000000..c5864ab --- /dev/null +++ ql/src/gen/vectorization/ExpressionTemplates/FilterColumnBetweenColumnAndScalar.txt @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + + +/** + * Generated from template FilterColumnBetween.txt, which covers [NOT] BETWEEN filter + * expressions where a column is [NOT] between one scalar and another. + * Output is not produced in a separate column. The selected vector of the input + * {@link VectorizedRowBatch} is updated for in-place filtering. + */ +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int colNum; + private int colNum1; + private rightValue; + + public (int colNum, int colNum1, rightValue) { + this.colNum = colNum; + this.colNum1 = colNum1; + this.rightValue = rightValue; + } + + public () { + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + if (childExpressions != null) { + super.evaluateChildren(batch); + } + inputColVector = () batch.cols[colNum]; + inputColVector1 = () batch.cols[colNum1]; + int[] sel = batch.selected; + int n = batch.size; + [] vector = inputColVector.vector; + [] vector1 = inputColVector1.vector; + boolean[] nullPos = inputColVector.isNull; + boolean[] nullPos1 = inputColVector1.isNull; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + int newSize = 0; + for (int j = 0; j != n; j++) { + int i = batch.selectedInUse ? sel[j] : j; + vi = inputColVector.isRepeating ? vector[0] : vector[i]; + v1i = inputColVector1.isRepeating ? vector1[0] : vector1[i]; + boolean ni = inputColVector.noNulls || (!inputColVector.noNulls && inputColVector.isRepeating && !nullPos[0]) || + (!inputColVector.noNulls && !inputColVector.isRepeating && !nullPos[i]); + boolean n1i = inputColVector1.noNulls || (!inputColVector1.noNulls && inputColVector1.isRepeating && !nullPos1[0]) || + (!inputColVector1.noNulls && !inputColVector1.isRepeating && !nullPos1[i]); + + if (ni && n1i && (vi >= v1i && vi <= rightValue)) { + sel[newSize++] = i; + } + } + + if (batch.selectedInUse) { + batch.size = newSize; + } else if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + + @Override + public int getOutputColumn() { + return -1; + } + + @Override + public String getOutputType() { + return "boolean"; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + public int getColNum1() { + return colNum1; + } + + public void setColNum1(int colNum1) { + this.colNum1 = colNum1; + } + + public getRightValue() { + return rightValue; + } + + public void setRightValue( value) { + this.rightValue = value; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.FILTER) + .setNumArguments(3) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string")) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); + } + +} diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterColumnBetweenScalarAndColumn.txt ql/src/gen/vectorization/ExpressionTemplates/FilterColumnBetweenScalarAndColumn.txt new file mode 100644 index 0000000..b61ce09 --- /dev/null +++ ql/src/gen/vectorization/ExpressionTemplates/FilterColumnBetweenScalarAndColumn.txt @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + +/** + * Generated from template FilterColumnBetween.txt, which covers [NOT] BETWEEN filter + * expressions where a column is [NOT] between one scalar and another. + * Output is not produced in a separate column. The selected vector of the input + * {@link VectorizedRowBatch} is updated for in-place filtering. + */ +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int colNum; + private leftValue; + private int colNum2; + + public (int colNum, leftValue, int colNum2) { + this.colNum = colNum; + this.leftValue = leftValue; + this.colNum2 = colNum2; + } + + public () { + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + if (childExpressions != null) { + super.evaluateChildren(batch); + } + inputColVector = () batch.cols[colNum]; + inputColVector2 = () batch.cols[colNum2]; + int[] sel = batch.selected; + int n = batch.size; + [] vector = inputColVector.vector; + [] vector2 = inputColVector2.vector; + boolean[] nullPos = inputColVector.isNull; + boolean[] nullPos2 = inputColVector2.isNull; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + int newSize = 0; + for (int j = 0; j != n; j++) { + int i = batch.selectedInUse ? sel[j] : j; + vi = inputColVector.isRepeating ? vector[0] : vector[i]; + v2i = inputColVector2.isRepeating ? vector2[0] : vector2[i]; + boolean ni = inputColVector.noNulls || (!inputColVector.noNulls && inputColVector.isRepeating && !nullPos[0]) || + (!inputColVector.noNulls && !inputColVector.isRepeating && !nullPos[i]); + boolean n2i = inputColVector2.noNulls || (!inputColVector2.noNulls && inputColVector2.isRepeating && !nullPos2[0]) || + (!inputColVector2.noNulls && !inputColVector2.isRepeating && !nullPos2[i]); + + if (ni && n2i && (vi >= leftValue && vi <= v2i)) { + sel[newSize++] = i; + } + } + + if (batch.selectedInUse) { + batch.size = newSize; + } else if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + + @Override + public int getOutputColumn() { + return -1; + } + + @Override + public String getOutputType() { + return "boolean"; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + public getLeftValue() { + return leftValue; + } + + public void setLeftValue( value) { + this.leftValue = value; + } + + public int getColNum2() { + return colNum2; + } + + public void setColNum2(int colNum2) { + this.colNum2 = colNum2; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.FILTER) + .setNumArguments(3) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string")) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } + +} diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetweenColumnAndColumn.txt ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetweenColumnAndColumn.txt new file mode 100644 index 0000000..d9881aa --- /dev/null +++ ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetweenColumnAndColumn.txt @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.common.type.Decimal128; +import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + + +/** + * Generated from template FilterColumnBetween.txt, which covers [NOT] BETWEEN filter + * expressions where a column is [NOT] between one scalar and another. + * Output is not produced in a separate column. The selected vector of the input + * {@link VectorizedRowBatch} is updated for in-place filtering. + */ +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int colNum; + private int colNum1; + private int colNum2; + + public (int colNum, int colNum1, int colNum2) { + this.colNum = colNum; + this.colNum1 = colNum1; + this.colNum2 = colNum2; + } + + public () { + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + if (childExpressions != null) { + super.evaluateChildren(batch); + } + DecimalColumnVector inputColVector = (DecimalColumnVector) batch.cols[colNum]; + DecimalColumnVector inputColVector1 = (DecimalColumnVector) batch.cols[colNum1]; + DecimalColumnVector inputColVector2 = (DecimalColumnVector) batch.cols[colNum2]; + int[] sel = batch.selected; + int n = batch.size; + Decimal128[] vector = inputColVector.vector; + Decimal128[] vector1 = inputColVector1.vector; + Decimal128[] vector2 = inputColVector2.vector; + boolean[] nullPos = inputColVector.isNull; + boolean[] nullPos1 = inputColVector1.isNull; + boolean[] nullPos2 = inputColVector2.isNull; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + int newSize = 0; + for (int j = 0; j != n; j++) { + int i = batch.selectedInUse ? sel[j] : j; + Decimal128 vi = inputColVector.isRepeating ? vector[0] : vector[i]; + Decimal128 v1i = inputColVector1.isRepeating ? vector1[0] : vector1[i]; + Decimal128 v2i = inputColVector2.isRepeating ? vector2[0] : vector2[i]; + boolean ni = inputColVector.noNulls || (!inputColVector.noNulls && inputColVector.isRepeating && !nullPos[0]) || + (!inputColVector.noNulls && !inputColVector.isRepeating && !nullPos[i]); + boolean n1i = inputColVector1.noNulls || (!inputColVector1.noNulls && inputColVector1.isRepeating && !nullPos1[0]) || + (!inputColVector1.noNulls && !inputColVector1.isRepeating && !nullPos1[i]); + boolean n2i = inputColVector2.noNulls || (!inputColVector2.noNulls && inputColVector2.isRepeating && !nullPos2[0]) || + (!inputColVector2.noNulls && !inputColVector2.isRepeating && !nullPos2[i]); + + if (ni && n1i && n2i && (vi.compareTo(v1i) >= 0 && vi.compareTo(v2i) <= 0)) { + sel[newSize++] = i; + } + } + + if (batch.selectedInUse) { + batch.size = newSize; + } else if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + + @Override + public int getOutputColumn() { + return -1; + } + + @Override + public String getOutputType() { + return "boolean"; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + public int getColNum1() { + return colNum1; + } + + public void setColNum1(int colNum1) { + this.colNum1 = colNum1; + } + + public int getColNum2() { + return colNum2; + } + + public void setColNum2(int colNum2) { + this.colNum2 = colNum2; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.FILTER) + .setNumArguments(3) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string")) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } + +} diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetweenColumnAndScalar.txt ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetweenColumnAndScalar.txt new file mode 100644 index 0000000..842961c --- /dev/null +++ ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetweenColumnAndScalar.txt @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.common.type.Decimal128; +import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + + +/** + * Generated from template FilterColumnBetween.txt, which covers [NOT] BETWEEN filter + * expressions where a column is [NOT] between one scalar and another. + * Output is not produced in a separate column. The selected vector of the input + * {@link VectorizedRowBatch} is updated for in-place filtering. + */ +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int colNum; + private int colNum1; + private Decimal128 rightValue; + + public (int colNum, int colNum1, Decimal128 rightValue) { + this.colNum = colNum; + this.colNum1 = colNum1; + this.rightValue = rightValue; + } + + public () { + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + if (childExpressions != null) { + super.evaluateChildren(batch); + } + DecimalColumnVector inputColVector = (DecimalColumnVector) batch.cols[colNum]; + DecimalColumnVector inputColVector1 = (DecimalColumnVector) batch.cols[colNum1]; + int[] sel = batch.selected; + int n = batch.size; + Decimal128[] vector = inputColVector.vector; + Decimal128[] vector1 = inputColVector1.vector; + boolean[] nullPos = inputColVector.isNull; + boolean[] nullPos1 = inputColVector1.isNull; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + int newSize = 0; + for (int j = 0; j != n; j++) { + int i = batch.selectedInUse ? sel[j] : j; + Decimal128 vi = inputColVector.isRepeating ? vector[0] : vector[i]; + Decimal128 v1i = inputColVector1.isRepeating ? vector1[0] : vector1[i]; + boolean ni = inputColVector.noNulls || (!inputColVector.noNulls && inputColVector.isRepeating && !nullPos[0]) || + (!inputColVector.noNulls && !inputColVector.isRepeating && !nullPos[i]); + boolean n1i = inputColVector1.noNulls || (!inputColVector1.noNulls && inputColVector1.isRepeating && !nullPos1[0]) || + (!inputColVector1.noNulls && !inputColVector1.isRepeating && !nullPos1[i]); + + if (ni && n1i && (vi.compareTo(v1i) >= 0 && vi.compareTo(rightValue) <= 0)) { + sel[newSize++] = i; + } + } + + if (batch.selectedInUse) { + batch.size = newSize; + } else if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + + @Override + public int getOutputColumn() { + return -1; + } + + @Override + public String getOutputType() { + return "boolean"; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + public int getColNum1() { + return colNum1; + } + + public void setColNum1(int colNum1) { + this.colNum1 = colNum1; + } + + public Decimal128 getRightValue() { + return rightValue; + } + + public void setRightValue(Decimal128 value) { + this.rightValue = value; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.FILTER) + .setNumArguments(3) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string")) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); + } + +} diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetweenScalarAndColumn.txt ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetweenScalarAndColumn.txt new file mode 100644 index 0000000..8e3ada5 --- /dev/null +++ ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetweenScalarAndColumn.txt @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.common.type.Decimal128; +import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + + +/** + * Generated from template FilterColumnBetween.txt, which covers [NOT] BETWEEN filter + * expressions where a column is [NOT] between one scalar and another. + * Output is not produced in a separate column. The selected vector of the input + * {@link VectorizedRowBatch} is updated for in-place filtering. + */ +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int colNum; + private int colNum2; + private Decimal128 leftValue; + + public (int colNum, Decimal128 leftValue, int colNum2) { + this.colNum = colNum; + this.colNum2 = colNum2; + this.leftValue = leftValue; + } + + public () { + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + if (childExpressions != null) { + super.evaluateChildren(batch); + } + DecimalColumnVector inputColVector = (DecimalColumnVector) batch.cols[colNum]; + DecimalColumnVector inputColVector2 = (DecimalColumnVector) batch.cols[colNum2]; + int[] sel = batch.selected; + int n = batch.size; + Decimal128[] vector = inputColVector.vector; + Decimal128[] vector2 = inputColVector2.vector; + boolean[] nullPos = inputColVector.isNull; + boolean[] nullPos2 = inputColVector2.isNull; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + int newSize = 0; + for (int j = 0; j != n; j++) { + int i = batch.selectedInUse ? sel[j] : j; + Decimal128 vi = inputColVector.isRepeating ? vector[0] : vector[i]; + Decimal128 v2i = inputColVector2.isRepeating ? vector2[0] : vector2[i]; + boolean ni = inputColVector.noNulls || (!inputColVector.noNulls && inputColVector.isRepeating && !nullPos[0]) || + (!inputColVector.noNulls && !inputColVector.isRepeating && !nullPos[i]); + boolean n2i = inputColVector2.noNulls || (!inputColVector2.noNulls && inputColVector2.isRepeating && !nullPos2[0]) || + (!inputColVector2.noNulls && !inputColVector2.isRepeating && !nullPos2[i]); + + if (ni && n2i && (vi.compareTo(leftValue) >= 0 && vi.compareTo(v2i) <= 0)) { + sel[newSize++] = i; + } + } + + if (batch.selectedInUse) { + batch.size = newSize; + } else if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + + @Override + public int getOutputColumn() { + return -1; + } + + @Override + public String getOutputType() { + return "boolean"; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + public Decimal128 getLeftValue() { + return leftValue; + } + + public void setLeftValue(Decimal128 value) { + this.leftValue = value; + } + + public int getColNum2() { + return colNum2; + } + + public void setColNum2(int colNum2) { + this.colNum2 = colNum2; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.FILTER) + .setNumArguments(3) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string")) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } + +} diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterStringColumnBetweenColumnAndColumn.txt ql/src/gen/vectorization/ExpressionTemplates/FilterStringColumnBetweenColumnAndColumn.txt new file mode 100644 index 0000000..6d89759 --- /dev/null +++ ql/src/gen/vectorization/ExpressionTemplates/FilterStringColumnBetweenColumnAndColumn.txt @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + + +/** + * This is a generated class to evaluate a [NOT] BETWEEN comparison on a vector of strings. + */ +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int colNum; + private int colNum1; + private int colNum2; + + public (int colNum, int colNum1, int colNum2) { + this.colNum = colNum; + this.colNum1 = colNum1; + this.colNum2 = colNum2; + } + + public () { + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + if (childExpressions != null) { + super.evaluateChildren(batch); + } + BytesColumnVector inputColVector = (BytesColumnVector) batch.cols[colNum]; + BytesColumnVector inputColVector1 = (BytesColumnVector) batch.cols[colNum1]; + BytesColumnVector inputColVector2 = (BytesColumnVector) batch.cols[colNum2]; + int[] sel = batch.selected; + int n = batch.size; + byte[][] vector = inputColVector.vector; + byte[][] vector1 = inputColVector1.vector; + byte[][] vector2 = inputColVector2.vector; + int[] length = inputColVector.length; + int[] start = inputColVector.start; + int[] start1 = inputColVector1.start; + int[] start2 = inputColVector2.start; + int[] length1 = inputColVector1.length; + int[] length2 = inputColVector2.length; + boolean[] nullPos = inputColVector.isNull; + boolean[] nullPos1 = inputColVector1.isNull; + boolean[] nullPos2 = inputColVector2.isNull; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + int newSize = 0; + for (int j = 0; j != n; j++) { + int i = batch.selectedInUse ? sel[j] : j; + byte[] vi = inputColVector.isRepeating ? vector[0] : vector[i]; + byte[] v1i = inputColVector1.isRepeating ? vector1[0] : vector1[i]; + byte[] v2i = inputColVector2.isRepeating ? vector2[0] : vector2[i]; + int si = inputColVector.isRepeating ? start[0] : start[i]; + int s1i = inputColVector1.isRepeating ? start1[0] : start1[i]; + int s2i = inputColVector2.isRepeating ? start2[0] : start2[i]; + int li = inputColVector.isRepeating ? length[0] : length[i]; + int l1i = inputColVector1.isRepeating ? length1[0] : length1[i]; + int l2i = inputColVector2.isRepeating ? length2[0] : length2[i]; + boolean ni = inputColVector.noNulls || (!inputColVector.noNulls && inputColVector.isRepeating && !nullPos[0]) || + (!inputColVector.noNulls && !inputColVector.isRepeating && !nullPos[i]); + boolean n1i = inputColVector1.noNulls || (!inputColVector1.noNulls && inputColVector1.isRepeating && !nullPos1[0]) || + (!inputColVector1.noNulls && !inputColVector1.isRepeating && !nullPos1[i]); + boolean n2i = inputColVector2.noNulls || (!inputColVector2.noNulls && inputColVector2.isRepeating && !nullPos2[0]) || + (!inputColVector2.noNulls && !inputColVector2.isRepeating && !nullPos2[i]); + + if (ni && n1i && n2i && (StringExpr.compare(vi, si, li, v1i, s1i, l1i) >= 0 && + StringExpr.compare(vi, si, li, v2i, s2i, l2i) <= 0)) { + sel[newSize++] = i; + } + } + + if (batch.selectedInUse) { + batch.size = newSize; + } else if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + + @Override + public int getOutputColumn() { + return -1; + } + + @Override + public String getOutputType() { + return "boolean"; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + public int getColNum1() { + return colNum1; + } + + public void setColNum1(int colNum1) { + this.colNum1 = colNum1; + } + + public int getColNum2() { + return colNum2; + } + + public void setColNum2(int colNum2) { + this.colNum2 = colNum2; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.FILTER) + .setNumArguments(3) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string")) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } + +} diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterStringColumnBetweenColumnAndScalar.txt ql/src/gen/vectorization/ExpressionTemplates/FilterStringColumnBetweenColumnAndScalar.txt new file mode 100644 index 0000000..a67dda4 --- /dev/null +++ ql/src/gen/vectorization/ExpressionTemplates/FilterStringColumnBetweenColumnAndScalar.txt @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + + +/** + * This is a generated class to evaluate a [NOT] BETWEEN comparison on a vector of strings. + */ +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int colNum; + private byte[] right; + private int colNum1; + + public (int colNum, int colNum1, byte[] right) { + this.colNum = colNum; + this.right = right; + this.colNum1 = colNum1; + } + + public () { + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + if (childExpressions != null) { + super.evaluateChildren(batch); + } + BytesColumnVector inputColVector = (BytesColumnVector) batch.cols[colNum]; + BytesColumnVector inputColVector1 = (BytesColumnVector) batch.cols[colNum1]; + int[] sel = batch.selected; + int n = batch.size; + byte[][] vector = inputColVector.vector; + byte[][] vector1 = inputColVector1.vector; + int[] length = inputColVector.length; + int[] start = inputColVector.start; + int[] start1 = inputColVector1.start; + int[] length1 = inputColVector1.length; + boolean[] nullPos = inputColVector.isNull; + boolean[] nullPos1 = inputColVector1.isNull; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + int newSize = 0; + for (int j = 0; j != n; j++) { + int i = batch.selectedInUse ? sel[j] : j; + byte[] vi = inputColVector.isRepeating ? vector[0] : vector[i]; + byte[] v1i = inputColVector1.isRepeating ? vector1[0] : vector1[i]; + int si = inputColVector.isRepeating ? start[0] : start[i]; + int s1i = inputColVector1.isRepeating ? start1[0] : start1[i]; + int li = inputColVector.isRepeating ? length[0] : length[i]; + int l1i = inputColVector1.isRepeating ? length1[0] : length1[i]; + boolean ni = inputColVector.noNulls || (!inputColVector.noNulls && inputColVector.isRepeating && !nullPos[0]) || + (!inputColVector.noNulls && !inputColVector.isRepeating && !nullPos[i]); + boolean n1i = inputColVector1.noNulls || (!inputColVector1.noNulls && inputColVector1.isRepeating && !nullPos1[0]) || + (!inputColVector1.noNulls && !inputColVector1.isRepeating && !nullPos1[i]); + + if (ni && n1i && (StringExpr.compare(vi, si, li, v1i, s1i, l1i) >= 0 && + StringExpr.compare(vi, si, li, right, 0, right.length) <= 0)) { + sel[newSize++] = i; + } + } + + if (batch.selectedInUse) { + batch.size = newSize; + } else if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + + @Override + public int getOutputColumn() { + return -1; + } + + @Override + public String getOutputType() { + return "boolean"; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + public byte[] getRight() { + return right; + } + + public void setRight(byte[] value) { + this.right = value; + } + + public int getColNum1() { + return colNum1; + } + + public void setColNum1(int colNum1) { + this.colNum1 = colNum1; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.FILTER) + .setNumArguments(3) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string")) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); + } + +} diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterStringColumnBetweenScalarAndColumn.txt ql/src/gen/vectorization/ExpressionTemplates/FilterStringColumnBetweenScalarAndColumn.txt new file mode 100644 index 0000000..f4131e2 --- /dev/null +++ ql/src/gen/vectorization/ExpressionTemplates/FilterStringColumnBetweenScalarAndColumn.txt @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + + +/** + * This is a generated class to evaluate a [NOT] BETWEEN comparison on a vector of strings. + */ +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int colNum; + private byte[] left; + private int colNum2; + + public (int colNum, byte[] left, int colNum2) { + this.colNum = colNum; + this.left = left; + this.colNum2 = colNum2; + } + + public () { + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + if (childExpressions != null) { + super.evaluateChildren(batch); + } + BytesColumnVector inputColVector = (BytesColumnVector) batch.cols[colNum]; + BytesColumnVector inputColVector2 = (BytesColumnVector) batch.cols[colNum2]; + int[] sel = batch.selected; + int n = batch.size; + byte[][] vector = inputColVector.vector; + byte[][] vector2 = inputColVector2.vector; + int[] length = inputColVector.length; + int[] start = inputColVector.start; + int[] start2 = inputColVector2.start; + int[] length2 = inputColVector2.length; + boolean[] nullPos = inputColVector.isNull; + boolean[] nullPos2 = inputColVector2.isNull; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + int newSize = 0; + for (int j = 0; j != n; j++) { + int i = batch.selectedInUse ? sel[j] : j; + byte[] vi = inputColVector.isRepeating ? vector[0] : vector[i]; + byte[] v2i = inputColVector2.isRepeating ? vector2[0] : vector2[i]; + int si = inputColVector.isRepeating ? start[0] : start[i]; + int s2i = inputColVector2.isRepeating ? start2[0] : start2[i]; + int li = inputColVector.isRepeating ? length[0] : length[i]; + int l2i = inputColVector2.isRepeating ? length2[0] : length2[i]; + boolean ni = inputColVector.noNulls || (!inputColVector.noNulls && inputColVector.isRepeating && !nullPos[0]) || + (!inputColVector.noNulls && !inputColVector.isRepeating && !nullPos[i]); + boolean n2i = inputColVector2.noNulls || (!inputColVector2.noNulls && inputColVector2.isRepeating && !nullPos2[0]) || + (!inputColVector2.noNulls && !inputColVector2.isRepeating && !nullPos2[i]); + + if (ni && n2i && (StringExpr.compare(vi, si, li, left, 0, left.length) >= 0 && + StringExpr.compare(vi, si, li, v2i, s2i, l2i) <= 0)) { + sel[newSize++] = i; + } + } + + if (batch.selectedInUse) { + batch.size = newSize; + } else if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + + @Override + public int getOutputColumn() { + return -1; + } + + @Override + public String getOutputType() { + return "boolean"; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + public byte[] getLeft() { + return left; + } + + public void setLeft(byte[] value) { + this.left = value; + } + + public int getColNum2() { + return colNum2; + } + + public void setColNum2(int colNum2) { + this.colNum2 = colNum2; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.FILTER) + .setNumArguments(3) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string"), + VectorExpressionDescriptor.ArgumentType.getType("string")) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.SCALAR, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index 535e4b3..87ea84d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -1282,43 +1282,154 @@ private VectorExpression getBetweenFilterExpression(List childExpr // prepare arguments for createVectorExpression List childrenAfterNot = foldConstantsForUnaryExprs(castChildren); - + + boolean lConstant = true; + boolean rConstant = true; + // The lhs of [not]between node should either be a constant or a column for vectorization. + if (childrenAfterNot.get(1) instanceof ExprNodeColumnDesc) { + lConstant = false; + } else if (!(childrenAfterNot.get(1) instanceof ExprNodeConstantDesc)) { + return null; + } + // The rhs of [not]between node should either be a constant or a column for vectorization. + if (childrenAfterNot.get(2) instanceof ExprNodeColumnDesc) { + rConstant = false; + } else if (!(childrenAfterNot.get(2) instanceof ExprNodeConstantDesc)) { + return null; + } // determine class Class cl = null; - if (isIntFamily(colType) && !notKeywordPresent) { - cl = FilterLongColumnBetween.class; - } else if (isIntFamily(colType) && notKeywordPresent) { - cl = FilterLongColumnNotBetween.class; - } else if (isFloatFamily(colType) && !notKeywordPresent) { - cl = FilterDoubleColumnBetween.class; - } else if (isFloatFamily(colType) && notKeywordPresent) { - cl = FilterDoubleColumnNotBetween.class; - } else if (colType.equals("string") && !notKeywordPresent) { - cl = FilterStringColumnBetween.class; - } else if (colType.equals("string") && notKeywordPresent) { - cl = FilterStringColumnNotBetween.class; - } else if (colType.equals("timestamp")) { - - // Get timestamp boundary values as longs instead of the expected strings - long left = getTimestampScalar(childExpr.get(2)); - long right = getTimestampScalar(childExpr.get(3)); - childrenAfterNot = new ArrayList(); - childrenAfterNot.add(colExpr); - childrenAfterNot.add(new ExprNodeConstantDesc(left)); - childrenAfterNot.add(new ExprNodeConstantDesc(right)); - if (notKeywordPresent) { - cl = FilterLongColumnNotBetween.class; - } else { - cl = FilterLongColumnBetween.class; - } - } else if (isDecimalFamily(colType) && !notKeywordPresent) { - cl = FilterDecimalColumnBetween.class; - } else if (isDecimalFamily(colType) && notKeywordPresent) { - cl = FilterDecimalColumnNotBetween.class; - } else if (isDateFamily(colType) && !notKeywordPresent) { - cl = FilterLongColumnBetween.class; - } else if (isDateFamily(colType) && notKeywordPresent) { - cl = FilterLongColumnNotBetween.class; + if (lConstant && rConstant) { + if (isIntFamily(colType) && !notKeywordPresent) { + cl = FilterLongColumnBetween.class; + } else if (isIntFamily(colType) && notKeywordPresent) { + cl = FilterLongColumnNotBetween.class; + } else if (isFloatFamily(colType) && !notKeywordPresent) { + cl = FilterDoubleColumnBetween.class; + } else if (isFloatFamily(colType) && notKeywordPresent) { + cl = FilterDoubleColumnNotBetween.class; + } else if (colType.equals("string") && !notKeywordPresent) { + cl = FilterStringColumnBetween.class; + } else if (colType.equals("string") && notKeywordPresent) { + cl = FilterStringColumnNotBetween.class; + } else if (colType.equals("timestamp")) { + // Get timestamp boundary values as longs instead of the expected strings + long left = getTimestampScalar(childExpr.get(2)); + long right = getTimestampScalar(childExpr.get(3)); + childrenAfterNot = new ArrayList(); + childrenAfterNot.add(colExpr); + childrenAfterNot.add(new ExprNodeConstantDesc(left)); + childrenAfterNot.add(new ExprNodeConstantDesc(right)); + if (notKeywordPresent) { + cl = FilterLongColumnNotBetween.class; + } else { + cl = FilterLongColumnBetween.class; + } + } else if (isDecimalFamily(colType) && !notKeywordPresent) { + cl = FilterDecimalColumnBetween.class; + } else if (isDecimalFamily(colType) && notKeywordPresent) { + cl = FilterDecimalColumnNotBetween.class; + } else if (isDateFamily(colType) && !notKeywordPresent) { + cl = FilterLongColumnBetween.class; + } else if (isDateFamily(colType) && notKeywordPresent) { + cl = FilterLongColumnNotBetween.class; + } + } else if (lConstant && !rConstant) { + if (isIntFamily(colType) && !notKeywordPresent) { + cl = FilterLongColumnBetweenScalarAndColumn.class; + } else if (isIntFamily(colType) && notKeywordPresent) { + cl = FilterLongColumnNotBetweenScalarAndColumn.class; + } else if (isFloatFamily(colType) && !notKeywordPresent) { + cl = FilterDoubleColumnBetweenScalarAndColumn.class; + } else if (isFloatFamily(colType) && notKeywordPresent) { + cl = FilterDoubleColumnNotBetweenScalarAndColumn.class; + } else if (colType.equals("string") && !notKeywordPresent) { + cl = FilterStringColumnBetweenScalarAndColumn.class; + } else if (colType.equals("string") && notKeywordPresent) { + cl = FilterStringColumnNotBetweenScalarAndColumn.class; + } else if (colType.equals("timestamp")) { + // Get timestamp boundary values as longs instead of the expected strings + long left = getTimestampScalar(childExpr.get(2)); + childrenAfterNot = new ArrayList(); + childrenAfterNot.add(colExpr); + childrenAfterNot.add(new ExprNodeConstantDesc(left)); + childrenAfterNot.add(new ExprNodeConstantDesc(childExpr.get(3))); + if (notKeywordPresent) { + cl = FilterLongColumnNotBetweenScalarAndColumn.class; + } else { + cl = FilterLongColumnBetweenScalarAndColumn.class; + } + } else if (isDecimalFamily(colType) && !notKeywordPresent) { + cl = FilterDecimalColumnBetweenScalarAndColumn.class; + } else if (isDecimalFamily(colType) && notKeywordPresent) { + cl = FilterDecimalColumnNotBetweenScalarAndColumn.class; + } else if (isDateFamily(colType) && !notKeywordPresent) { + cl = FilterLongColumnBetweenScalarAndColumn.class; + } else if (isDateFamily(colType) && notKeywordPresent) { + cl = FilterLongColumnNotBetweenScalarAndColumn.class; + } + } else if (!lConstant && rConstant) { + if (isIntFamily(colType) && !notKeywordPresent) { + cl = FilterLongColumnBetweenColumnAndScalar.class; + } else if (isIntFamily(colType) && notKeywordPresent) { + cl = FilterLongColumnNotBetweenColumnAndScalar.class; + } else if (isFloatFamily(colType) && !notKeywordPresent) { + cl = FilterDoubleColumnBetweenColumnAndScalar.class; + } else if (isFloatFamily(colType) && notKeywordPresent) { + cl = FilterDoubleColumnNotBetweenColumnAndScalar.class; + } else if (colType.equals("string") && !notKeywordPresent) { + cl = FilterStringColumnBetweenColumnAndScalar.class; + } else if (colType.equals("string") && notKeywordPresent) { + cl = FilterStringColumnNotBetweenColumnAndScalar.class; + } else if (colType.equals("timestamp")) { + // Get timestamp boundary values as longs instead of the expected strings + long right = getTimestampScalar(childExpr.get(3)); + childrenAfterNot = new ArrayList(); + childrenAfterNot.add(colExpr); + childrenAfterNot.add(childExpr.get(2)); + childrenAfterNot.add(new ExprNodeConstantDesc(right)); + if (notKeywordPresent) { + cl = FilterLongColumnNotBetweenColumnAndScalar.class; + } else { + cl = FilterLongColumnBetweenColumnAndScalar.class; + } + } else if (isDecimalFamily(colType) && !notKeywordPresent) { + cl = FilterDecimalColumnBetweenColumnAndScalar.class; + } else if (isDecimalFamily(colType) && notKeywordPresent) { + cl = FilterDecimalColumnNotBetweenColumnAndScalar.class; + } else if (isDateFamily(colType) && !notKeywordPresent) { + cl = FilterLongColumnBetweenColumnAndScalar.class; + } else if (isDateFamily(colType) && notKeywordPresent) { + cl = FilterLongColumnNotBetweenColumnAndScalar.class; + } + } else { + if (isIntFamily(colType) && !notKeywordPresent) { + cl = FilterLongColumnBetweenColumnAndColumn.class; + } else if (isIntFamily(colType) && notKeywordPresent) { + cl = FilterLongColumnNotBetweenColumnAndColumn.class; + } else if (isFloatFamily(colType) && !notKeywordPresent) { + cl = FilterDoubleColumnBetweenColumnAndColumn.class; + } else if (isFloatFamily(colType) && notKeywordPresent) { + cl = FilterDoubleColumnNotBetweenColumnAndColumn.class; + } else if (colType.equals("string") && !notKeywordPresent) { + cl = FilterStringColumnBetweenColumnAndColumn.class; + } else if (colType.equals("string") && notKeywordPresent) { + cl = FilterStringColumnNotBetweenColumnAndColumn.class; + } else if (colType.equals("timestamp")) { + if (notKeywordPresent) { + cl = FilterLongColumnNotBetweenColumnAndColumn.class; + } else { + cl = FilterLongColumnBetweenColumnAndColumn.class; + } + } else if (isDecimalFamily(colType) && !notKeywordPresent) { + cl = FilterDecimalColumnBetweenColumnAndColumn.class; + } else if (isDecimalFamily(colType) && notKeywordPresent) { + cl = FilterDecimalColumnNotBetweenColumnAndColumn.class; + } else if (isDateFamily(colType) && !notKeywordPresent) { + cl = FilterLongColumnBetweenColumnAndColumn.class; + } else if (isDateFamily(colType) && notKeywordPresent) { + cl = FilterLongColumnNotBetweenColumnAndColumn.class; + } } return createVectorExpression(cl, childrenAfterNot, Mode.PROJECTION, returnType); } diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java index 2329f52..c3ea9b7 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java @@ -61,6 +61,18 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.FilterStringColumnInList; import org.apache.hadoop.hive.ql.exec.vector.expressions.FilterLongColumnInList; import org.apache.hadoop.hive.ql.exec.vector.expressions.FilterDoubleColumnInList; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnBetweenColumnAndColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnBetweenColumnAndScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnBetweenScalarAndColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnNotBetweenColumnAndColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnNotBetweenColumnAndScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnNotBetweenScalarAndColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnBetweenColumnAndColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnBetweenColumnAndScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnBetweenScalarAndColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnNotBetweenColumnAndColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnNotBetweenColumnAndScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnNotBetweenScalarAndColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongColumnLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongColumnLongScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongScalarLongScalar; @@ -911,7 +923,9 @@ public void testBetweenFilters() throws HiveException { ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(String.class, "col1", "table", false); ExprNodeConstantDesc constDesc = new ExprNodeConstantDesc("Alpha"); ExprNodeConstantDesc constDesc2 = new ExprNodeConstantDesc("Bravo"); - + ExprNodeColumnDesc col2Expr = new ExprNodeColumnDesc(String.class, "col3", "table", false); + ExprNodeColumnDesc col3Expr = new ExprNodeColumnDesc(String.class, "col4", "table", false); + // string BETWEEN GenericUDFBetween udf = new GenericUDFBetween(); List children1 = new ArrayList(); @@ -921,19 +935,76 @@ public void testBetweenFilters() throws HiveException { children1.add(constDesc2); ExprNodeGenericFuncDesc exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, children1); - + + //ColumnAndColumn + List children2 = new ArrayList(); + children2.add(new ExprNodeConstantDesc(new Boolean(false))); // no NOT keyword + children2.add(col1Expr); + children2.add(col2Expr); + children2.add(col3Expr); + ExprNodeGenericFuncDesc exprDesc2 = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, + children2); + + //ScalarAndColumn + List children3 = new ArrayList(); + children3.add(new ExprNodeConstantDesc(new Boolean(false))); // no NOT keyword + children3.add(col1Expr); + children3.add(constDesc); + children3.add(col3Expr); + ExprNodeGenericFuncDesc exprDesc3 = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, + children3); + + //ColumnAndScalar + List children4 = new ArrayList(); + children4.add(new ExprNodeConstantDesc(new Boolean(false))); // no NOT keyword + children4.add(col1Expr); + children4.add(col2Expr); + children4.add(constDesc); + ExprNodeGenericFuncDesc exprDesc4 = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, + children4); + + Map columnMap = new HashMap(); - columnMap.put("col1", 1); - columnMap.put("col2", 2); - VectorizationContext vc = new VectorizationContext(columnMap, 2); + columnMap.put("col1", 0); + columnMap.put("col2", 1); + columnMap.put("col3", 2); + columnMap.put("col4", 3); + VectorizationContext vc = new VectorizationContext(columnMap, 3); VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); assertTrue(ve instanceof FilterStringColumnBetween); + + VectorExpression ve2 = vc.getVectorExpression(exprDesc2, VectorExpressionDescriptor.Mode.FILTER); + assertTrue(ve2 instanceof FilterStringColumnBetweenColumnAndColumn); + + VectorExpression ve3 = vc.getVectorExpression(exprDesc3, VectorExpressionDescriptor.Mode.FILTER); + assertTrue(ve3 instanceof FilterStringColumnBetweenScalarAndColumn); + + VectorExpression ve4 = vc.getVectorExpression(exprDesc4, VectorExpressionDescriptor.Mode.FILTER); + assertTrue(ve4 instanceof FilterStringColumnBetweenColumnAndScalar); // string NOT BETWEEN children1.set(0, new ExprNodeConstantDesc(new Boolean(true))); // has NOT keyword ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); assertTrue(ve instanceof FilterStringColumnNotBetween); - + + children2.set(0, new ExprNodeConstantDesc(new Boolean(true))); // has NOT keyword + exprDesc2 = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, + children2); + ve2 = vc.getVectorExpression(exprDesc2, VectorExpressionDescriptor.Mode.FILTER); + assertTrue(ve2 instanceof FilterStringColumnNotBetweenColumnAndColumn); + + children3.set(0, new ExprNodeConstantDesc(new Boolean(true))); // has NOT keyword + exprDesc3 = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, + children3); + ve3 = vc.getVectorExpression(exprDesc3, VectorExpressionDescriptor.Mode.FILTER); + assertTrue(ve3 instanceof FilterStringColumnNotBetweenScalarAndColumn); + + children4.set(0, new ExprNodeConstantDesc(new Boolean(true))); // has NOT keyword + exprDesc4 = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, + children4); + ve4 = vc.getVectorExpression(exprDesc4, VectorExpressionDescriptor.Mode.FILTER); + assertTrue(ve4 instanceof FilterStringColumnNotBetweenColumnAndScalar); + // long BETWEEN children1.set(0, new ExprNodeConstantDesc(new Boolean(false))); children1.set(1, new ExprNodeColumnDesc(Long.class, "col1", "table", false)); @@ -942,10 +1013,65 @@ public void testBetweenFilters() throws HiveException { ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); assertTrue(ve instanceof FilterLongColumnBetween); + children2.set(0, new ExprNodeConstantDesc(new Boolean(false))); + children2.set(1, new ExprNodeColumnDesc(Long.class, "col1", "table", false)); + children2.set(2, new ExprNodeColumnDesc(Long.class, "col2", "table", false)); + children2.set(3, new ExprNodeColumnDesc(Long.class, "col3", "table", false)); + exprDesc2 = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, + children2); + ve2 = vc.getVectorExpression(exprDesc2, VectorExpressionDescriptor.Mode.FILTER); + assertTrue(ve2 instanceof FilterLongColumnBetweenColumnAndColumn); + + children3.set(0, new ExprNodeConstantDesc(new Boolean(false))); + children3.set(1, new ExprNodeColumnDesc(Long.class, "col1", "table", false)); + children3.set(2, new ExprNodeConstantDesc(10)); + children3.set(3, new ExprNodeColumnDesc(Long.class, "col3", "table", false)); + exprDesc3 = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, + children3); + ve3 = vc.getVectorExpression(exprDesc3, VectorExpressionDescriptor.Mode.FILTER); + assertTrue(ve3 instanceof FilterLongColumnBetweenScalarAndColumn); + + children4.set(0, new ExprNodeConstantDesc(new Boolean(false))); + children4.set(1, new ExprNodeColumnDesc(Long.class, "col1", "table", false)); + children4.set(2, new ExprNodeColumnDesc(Long.class, "col3", "table", false)); + children4.set(3, new ExprNodeConstantDesc(10)); + exprDesc4 = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, + children4); + ve4 = vc.getVectorExpression(exprDesc4, VectorExpressionDescriptor.Mode.FILTER); + assertTrue(ve4 instanceof FilterLongColumnBetweenColumnAndScalar); + + // long NOT BETWEEN children1.set(0, new ExprNodeConstantDesc(new Boolean(true))); ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); assertTrue(ve instanceof FilterLongColumnNotBetween); + + children2.set(0, new ExprNodeConstantDesc(new Boolean(true))); + children2.set(1, new ExprNodeColumnDesc(Long.class, "col1", "table", false)); + children2.set(2, new ExprNodeColumnDesc(Long.class, "col2", "table", false)); + children2.set(3, new ExprNodeColumnDesc(Long.class, "col3", "table", false)); + exprDesc2 = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, + children2); + ve2 = vc.getVectorExpression(exprDesc2, VectorExpressionDescriptor.Mode.FILTER); + assertTrue(ve2 instanceof FilterLongColumnNotBetweenColumnAndColumn); + + children3.set(0, new ExprNodeConstantDesc(new Boolean(true))); + children3.set(1, new ExprNodeColumnDesc(Long.class, "col1", "table", false)); + children3.set(2, new ExprNodeConstantDesc(10)); + children3.set(3, new ExprNodeColumnDesc(Long.class, "col3", "table", false)); + exprDesc3 = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, + children3); + ve3 = vc.getVectorExpression(exprDesc3, VectorExpressionDescriptor.Mode.FILTER); + assertTrue(ve3 instanceof FilterLongColumnNotBetweenScalarAndColumn); + + children4.set(0, new ExprNodeConstantDesc(new Boolean(true))); + children4.set(1, new ExprNodeColumnDesc(Long.class, "col1", "table", false)); + children4.set(2, new ExprNodeColumnDesc(Long.class, "col3", "table", false)); + children4.set(3, new ExprNodeConstantDesc(10)); + exprDesc4 = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, + children4); + ve4 = vc.getVectorExpression(exprDesc4, VectorExpressionDescriptor.Mode.FILTER); + assertTrue(ve4 instanceof FilterLongColumnNotBetweenColumnAndScalar); // double BETWEEN children1.set(0, new ExprNodeConstantDesc(new Boolean(false))); @@ -954,7 +1080,7 @@ public void testBetweenFilters() throws HiveException { children1.set(3, new ExprNodeConstantDesc(20.0d)); ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); assertTrue(ve instanceof FilterDoubleColumnBetween); - + // double NOT BETWEEN children1.set(0, new ExprNodeConstantDesc(new Boolean(true))); ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java index 92e5a06..e191497 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java @@ -41,7 +41,13 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColGreaterLongScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColLessLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnBetween; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnBetweenColumnAndColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnBetweenColumnAndScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnBetweenScalarAndColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnNotBetween; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnNotBetweenColumnAndColumn; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnNotBetweenColumnAndScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnNotBetweenScalarAndColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongScalarGreaterLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongScalarLessLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnBetween; @@ -448,6 +454,308 @@ public void testFilterLongNotBetween() { } @Test + public void testFilterLongBetweenColumnAndColumn() { + int seed = 17; + VectorizedRowBatch vrb = VectorizedRowGroupGenUtil.getVectorizedRowBatch( + 5, 4, seed); + LongColumnVector lcv0 = (LongColumnVector) vrb.cols[0]; + LongColumnVector lcv1 = (LongColumnVector) vrb.cols[1]; + LongColumnVector lcv2 = (LongColumnVector) vrb.cols[2]; + VectorExpression expr1 = new FilterLongColumnBetweenColumnAndColumn(0, 1, 2); + + //Basic case + lcv0.vector[0] = 5; + lcv0.vector[1] = 20; + lcv0.vector[2] = 17; + lcv0.vector[3] = 15; + lcv0.vector[4] = 10; + + lcv1.vector[0] = 3; + lcv1.vector[1] = 21; + lcv1.vector[2] = 10; + lcv1.vector[3] = 14; + lcv1.vector[4] = 1; + + lcv2.vector[0] = 4; + lcv2.vector[1] = 29; + lcv2.vector[2] = 25; + lcv2.vector[3] = 18; + lcv2.vector[4] = 1; + + expr1.evaluate(vrb); + + assertEquals(2, vrb.size); + assertTrue(vrb.selectedInUse); + assertEquals(2, vrb.selected[0]); + assertEquals(3, vrb.selected[1]); + + //With nulls + VectorizedRowBatch vrb1 = VectorizedRowGroupGenUtil.getVectorizedRowBatch( + 5, 4, seed); + + lcv0 = (LongColumnVector) vrb1.cols[0]; + lcv1 = (LongColumnVector) vrb1.cols[1]; + lcv2 = (LongColumnVector) vrb1.cols[2]; + + lcv0.vector[0] = 5; + lcv0.vector[1] = 20; + lcv0.vector[2] = 17; + lcv0.vector[3] = 15; + lcv0.vector[4] = 10; + + lcv0.noNulls = false; + lcv0.isNull[0] = true; + lcv0.isNull[2] = true; + + lcv1.vector[0] = 3; + lcv1.vector[1] = 21; + lcv1.vector[2] = 10; + lcv1.vector[3] = 14; + lcv1.vector[4] = 1; + + lcv2.vector[0] = 4; + lcv2.vector[1] = 29; + lcv2.vector[2] = 25; + lcv2.vector[3] = 18; + lcv2.vector[4] = 1; + + + expr1.evaluate(vrb1); + assertEquals(1, vrb1.size); + assertTrue(vrb1.selectedInUse); + assertEquals(3, vrb1.selected[0]); + + //With nulls and selected + VectorizedRowBatch vrb2 = VectorizedRowGroupGenUtil.getVectorizedRowBatch( + 7, 4, seed); + vrb2.selectedInUse = true; + vrb2.selected[0] = 1; + vrb2.selected[1] = 2; + vrb2.selected[2] = 4; + vrb2.size = 3; + + lcv0 = (LongColumnVector) vrb2.cols[0]; + lcv1 = (LongColumnVector) vrb2.cols[1]; + lcv2 = (LongColumnVector) vrb2.cols[2]; + + + lcv0.vector[0] = 5; + lcv0.vector[1] = 20; + lcv0.vector[2] = 17; + lcv0.vector[3] = 15; + lcv0.vector[4] = 10; + lcv0.vector[5] = 19; + lcv0.vector[6] = 21; + + lcv0.noNulls = false; + lcv0.isNull[0] = true; + lcv0.isNull[2] = true; + lcv0.isNull[5] = true; + + lcv1.vector[0] = 3; + lcv1.vector[1] = 21; + lcv1.vector[2] = 10; + lcv1.vector[3] = 14; + lcv1.vector[4] = 1; + + lcv2.vector[0] = 4; + lcv2.vector[1] = 29; + lcv2.vector[2] = 25; + lcv2.vector[3] = 18; + lcv2.vector[4] = 1; + + + expr1.evaluate(vrb2); + assertEquals(0, vrb2.size); + + //Repeating non null + VectorizedRowBatch vrb3 = VectorizedRowGroupGenUtil.getVectorizedRowBatch( + 7, 4, seed); + lcv0 = (LongColumnVector) vrb3.cols[0]; + lcv1 = (LongColumnVector) vrb3.cols[1]; + lcv2 = (LongColumnVector) vrb3.cols[2]; + + lcv0.isRepeating = true; + lcv0.vector[0] = 17; + + expr1.evaluate(vrb3); + assertEquals(0, vrb3.size); + assertTrue(lcv0.isRepeating); + + //Repeating null + lcv0.noNulls = false; + lcv0.vector[0] = 17; + lcv0.isNull[0] = true; + + lcv1.vector[0] = 3; + lcv1.vector[1] = 21; + lcv1.vector[2] = 10; + lcv1.vector[3] = 14; + lcv1.vector[4] = 1; + + lcv2.vector[0] = 4; + lcv2.vector[1] = 29; + lcv2.vector[2] = 25; + lcv2.vector[3] = 18; + lcv2.vector[4] = 1; + + expr1.evaluate(vrb3); + assertEquals(0, vrb3.size); + + } + + @Test + public void testFilterLongNotBetweenColumnAndColumn() { + + // Spot check only. null & repeating behavior are checked elsewhere for the same template. + + VectorizedRowBatch vrb = VectorizedRowGroupGenUtil.getVectorizedRowBatch( + 5, 4, 17); + LongColumnVector lcv0 = (LongColumnVector) vrb.cols[0]; + LongColumnVector lcv1 = (LongColumnVector) vrb.cols[1]; + LongColumnVector lcv2 = (LongColumnVector) vrb.cols[2]; + + //Basic case + lcv0.vector[0] = 5; + lcv0.vector[1] = 20; + lcv0.vector[2] = 17; + lcv0.vector[3] = 15; + lcv0.vector[4] = 10; + + lcv1.vector[0] = 3; + lcv1.vector[1] = 21; + lcv1.vector[2] = 10; + lcv1.vector[3] = 14; + lcv1.vector[4] = 1; + + lcv2.vector[0] = 4; + lcv2.vector[1] = 29; + lcv2.vector[2] = 25; + lcv2.vector[3] = 18; + lcv2.vector[4] = 1; + + VectorExpression expr = new FilterLongColumnNotBetweenColumnAndColumn(0, 1, 2); + expr.evaluate(vrb); + assertEquals(3, vrb.size); + assertTrue(vrb.selectedInUse); + assertEquals(0, vrb.selected[0]); + } + + @Test + public void testFilterLongBetweenScalarAndColumn() { + + // Spot check only. null & repeating behavior are checked elsewhere for the same template. + VectorizedRowBatch vrb = VectorizedRowGroupGenUtil.getVectorizedRowBatch( + 5, 4, 17); + LongColumnVector lcv0 = (LongColumnVector) vrb.cols[0]; + LongColumnVector lcv2 = (LongColumnVector) vrb.cols[2]; + + //Basic case + lcv0.vector[0] = 5; + lcv0.vector[1] = 20; + lcv0.vector[2] = 17; + lcv0.vector[3] = 15; + lcv0.vector[4] = 10; + + lcv2.vector[0] = 4; + lcv2.vector[1] = 29; + lcv2.vector[2] = 25; + lcv2.vector[3] = 18; + lcv2.vector[4] = 1; + + VectorExpression expr = new FilterLongColumnBetweenScalarAndColumn(0, 4, 2); + expr.evaluate(vrb); + assertEquals(3, vrb.size); + assertTrue(vrb.selectedInUse); + assertEquals(1, vrb.selected[0]); + } + + @Test + public void testFilterLongNotBetweenScalarAndColumn() { + + // Spot check only. null & repeating behavior are checked elsewhere for the same template. + VectorizedRowBatch vrb = VectorizedRowGroupGenUtil.getVectorizedRowBatch( + 5, 4, 17); + LongColumnVector lcv0 = (LongColumnVector) vrb.cols[0]; + LongColumnVector lcv2 = (LongColumnVector) vrb.cols[2]; + + //Basic case + lcv0.vector[0] = 5; + lcv0.vector[1] = 20; + lcv0.vector[2] = 17; + lcv0.vector[3] = 15; + lcv0.vector[4] = 10; + + lcv2.vector[0] = 4; + lcv2.vector[1] = 29; + lcv2.vector[2] = 25; + lcv2.vector[3] = 18; + lcv2.vector[4] = 1; + + VectorExpression expr = new FilterLongColumnNotBetweenScalarAndColumn(0, 4, 2); + expr.evaluate(vrb); + assertEquals(2, vrb.size); + assertTrue(vrb.selectedInUse); + } + + @Test + public void testFilterLongBetweenColumnAndScalar() { + + // Spot check only. null & repeating behavior are checked elsewhere for the same template. + VectorizedRowBatch vrb = VectorizedRowGroupGenUtil.getVectorizedRowBatch( + 5, 4, 17); + LongColumnVector lcv0 = (LongColumnVector) vrb.cols[0]; + LongColumnVector lcv2 = (LongColumnVector) vrb.cols[2]; + + //Basic case + lcv0.vector[0] = 5; + lcv0.vector[1] = 20; + lcv0.vector[2] = 17; + lcv0.vector[3] = 15; + lcv0.vector[4] = 10; + + lcv2.vector[0] = 4; + lcv2.vector[1] = 29; + lcv2.vector[2] = 25; + lcv2.vector[3] = 18; + lcv2.vector[4] = 1; + + VectorExpression expr = new FilterLongColumnBetweenColumnAndScalar(0, 2, 50); + expr.evaluate(vrb); + assertEquals(2, vrb.size); + assertTrue(vrb.selectedInUse); + assertEquals(0, vrb.selected[0]); + } + + @Test + public void testFilterLongNotBetweenColumnAndScalar() { + + // Spot check only. null & repeating behavior are checked elsewhere for the same template. + VectorizedRowBatch vrb = VectorizedRowGroupGenUtil.getVectorizedRowBatch( + 5, 4, 17); + LongColumnVector lcv0 = (LongColumnVector) vrb.cols[0]; + LongColumnVector lcv2 = (LongColumnVector) vrb.cols[2]; + + //Basic case + lcv0.vector[0] = 5; + lcv0.vector[1] = 20; + lcv0.vector[2] = 17; + lcv0.vector[3] = 15; + lcv0.vector[4] = 10; + + lcv2.vector[0] = 4; + lcv2.vector[1] = 29; + lcv2.vector[2] = 25; + lcv2.vector[3] = 18; + lcv2.vector[4] = 1; + + VectorExpression expr = new FilterLongColumnNotBetweenColumnAndScalar(0, 2, 50); + expr.evaluate(vrb); + assertEquals(3, vrb.size); + assertTrue(vrb.selectedInUse); + } + + @Test public void testFilterDoubleBetween() { // Spot check only. null & repeating behavior are checked elsewhere for the same template.