From c3060eb212300feef399c1260b0fd01e6395e5fa Mon Sep 17 00:00:00 2001 From: Tony Murphy Date: Thu, 16 May 2013 15:48:03 -0700 Subject: [PATCH] HIVE-4553 --- .../exec/vector/expressions/templates/CodeGen.java | 116 +++++++--- .../vector/expressions/templates/TestClass.txt | 44 ++++ .../vector/expressions/templates/TestCodeGen.java | 256 +++++++++++++++++++++ ...olumnColumnFilterVectorExpressionEvaluation.txt | 68 ++++++ ...mnColumnOperationVectorExpressionEvaluation.txt | 60 +++++ ...olumnScalarFilterVectorExpressionEvaluation.txt | 73 ++++++ ...mnScalarOperationVectorExpressionEvaluation.txt | 58 +++++ .../vector/util/VectorizedRowGroupGenUtil.java | 74 ++++++ 8 files changed, 720 insertions(+), 29 deletions(-) create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java index 53d9a7a..2ce242a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java @@ -216,10 +216,12 @@ }; + private final String templateDirectory; private final String outputDirectory; + private final TestCodeGen testCodeGen; - private static String joinPath(String...parts) { + static String joinPath(String...parts) { String path = parts[0]; for (int i=1; i < parts.length; ++i) { path += File.separatorChar + parts[i]; @@ -231,11 +233,13 @@ public CodeGen() { templateDirectory = System.getProperty("user.dir"); File f = new File(templateDirectory); outputDirectory = joinPath(f.getParent(), "gen"); + testCodeGen = new TestCodeGen(joinPath(f.getParent(), "test"),templateDirectory); } - public CodeGen(String templateDirectory, String outputDirectory) { + public CodeGen(String templateDirectory, String outputDirectory, String testOutputDirectory) { this.templateDirectory = templateDirectory; this.outputDirectory = outputDirectory; + testCodeGen = new TestCodeGen(testOutputDirectory,templateDirectory); } /** @@ -244,10 +248,13 @@ public CodeGen(String templateDirectory, String outputDirectory) { */ public static void main(String[] args) throws Exception { CodeGen gen; - if (args != null && args[0] != null) { - gen = new CodeGen(args[0], args[1]); - } else { + if (args == null || args.length==0) { gen = new CodeGen(); + } else if (args.length==3) { + gen = new CodeGen(args[0], args[1], args[2]); + }else{ + System.out.println("args: "); + return; } gen.generate(); } @@ -277,6 +284,7 @@ private void generate() throws Exception { continue; } } + testCodeGen.generateTestSuites(); } private void generateVectorUDAFMinMax(String[] tdesc) throws Exception { @@ -407,9 +415,9 @@ private void generateFilterColumnCompareColumn(String[] tdesc) throws IOExceptio String operatorName = tdesc[1]; String operandType1 = tdesc[2]; String operandType2 = tdesc[3]; - String className = "Filter" + this.getCamelCaseType(operandType1) - + "Col" + operatorName + this.getCamelCaseType(operandType2) + "Column"; - generateColumnBinaryOperatorColumn(tdesc, "doesn't matter", className); + String className = "Filter" + getCamelCaseType(operandType1) + + "Col" + operatorName + getCamelCaseType(operandType2) + "Column"; + generateColumnBinaryOperatorColumn(tdesc, null, className); } private void generateColumnUnaryMinus(String[] tdesc) throws IOException { @@ -417,7 +425,7 @@ private void generateColumnUnaryMinus(String[] tdesc) throws IOException { String inputColumnVectorType = this.getColumnVectorType(operandType); String outputColumnVectorType = inputColumnVectorType; String returnType = operandType; - String className = this.getCamelCaseType(operandType) + "ColUnaryMinus"; + String className = getCamelCaseType(operandType) + "ColUnaryMinus"; String outputFile = joinPath(this.outputDirectory, className + ".java"); String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); String templateString = readFile(templateFile); @@ -434,8 +442,8 @@ private void generateColumnArithmeticColumn(String [] tdesc) throws IOException String operatorName = tdesc[1]; String operandType1 = tdesc[2]; String operandType2 = tdesc[3]; - String className = this.getCamelCaseType(operandType1) - + "Col" + operatorName + this.getCamelCaseType(operandType2) + "Column"; + String className = getCamelCaseType(operandType1) + + "Col" + operatorName + getCamelCaseType(operandType2) + "Column"; String returnType = getArithmeticReturnType(operandType1, operandType2); generateColumnBinaryOperatorColumn(tdesc, returnType, className); } @@ -446,9 +454,9 @@ private void generateFilterColumnCompareScalar(String[] tdesc) throws IOExceptio String operatorName = tdesc[1]; String operandType1 = tdesc[2]; String operandType2 = tdesc[3]; - String className = "Filter" + this.getCamelCaseType(operandType1) - + "Col" + operatorName + this.getCamelCaseType(operandType2) + "Scalar"; - generateColumnBinaryOperatorScalar(tdesc, "doesn't matter", className); + String className = "Filter" + getCamelCaseType(operandType1) + + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar"; + generateColumnBinaryOperatorScalar(tdesc, null, className); } private void generateFilterScalarCompareColumn(String[] tdesc) throws IOException { @@ -456,9 +464,9 @@ private void generateFilterScalarCompareColumn(String[] tdesc) throws IOExceptio String operatorName = tdesc[1]; String operandType1 = tdesc[2]; String operandType2 = tdesc[3]; - String className = "Filter" + this.getCamelCaseType(operandType1) - + "Scalar" + operatorName + this.getCamelCaseType(operandType2) + "Column"; - generateScalarBinaryOperatorColumn(tdesc, "doesn't matter", className); + String className = "Filter" + getCamelCaseType(operandType1) + + "Scalar" + operatorName + getCamelCaseType(operandType2) + "Column"; + generateScalarBinaryOperatorColumn(tdesc, null, className); } private void generateColumnCompareScalar(String[] tdesc) throws IOException { @@ -466,8 +474,8 @@ private void generateColumnCompareScalar(String[] tdesc) throws IOException { String operandType1 = tdesc[2]; String operandType2 = tdesc[3]; String returnType = "long"; - String className = this.getCamelCaseType(operandType1) - + "Col" + operatorName + this.getCamelCaseType(operandType2) + "Scalar"; + String className = getCamelCaseType(operandType1) + + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar"; generateColumnBinaryOperatorScalar(tdesc, returnType, className); } @@ -493,6 +501,20 @@ private void generateColumnBinaryOperatorColumn(String[] tdesc, String returnTyp templateString = templateString.replaceAll("", operandType2); templateString = templateString.replaceAll("", returnType); writeFile(outputFile, templateString); + + if(returnType==null){ + testCodeGen.addColumnColumnFilterTestCases( + className, + inputColumnVectorType1, + inputColumnVectorType2, + operatorSymbol); + }else{ + testCodeGen.addColumnColumnOperationTestCases( + className, + inputColumnVectorType1, + inputColumnVectorType2, + outputColumnVectorType); + } } private void generateColumnBinaryOperatorScalar(String[] tdesc, String returnType, @@ -515,6 +537,25 @@ private void generateColumnBinaryOperatorScalar(String[] tdesc, String returnTyp templateString = templateString.replaceAll("", operandType2); templateString = templateString.replaceAll("", returnType); writeFile(outputFile, templateString); + + if(returnType==null) + { + testCodeGen.addColumnScalarFilterTestCases( + true, + className, + inputColumnVectorType, + operandType2, + operatorSymbol); + }else + { + testCodeGen.addColumnScalarOperationTestCases( + true, + className, + inputColumnVectorType, + outputColumnVectorType, + operandType2); + } + } private void generateScalarBinaryOperatorColumn(String[] tdesc, String returnType, @@ -537,6 +578,24 @@ private void generateScalarBinaryOperatorColumn(String[] tdesc, String returnTyp templateString = templateString.replaceAll("", operandType2); templateString = templateString.replaceAll("", returnType); writeFile(outputFile, templateString); + + if(returnType==null) + { + testCodeGen.addColumnScalarFilterTestCases( + false, + className, + inputColumnVectorType, + operandType1, + operatorSymbol); + }else + { + testCodeGen.addColumnScalarOperationTestCases( + false, + className, + inputColumnVectorType, + outputColumnVectorType, + operandType1); + } } //Binary arithmetic operator @@ -544,8 +603,8 @@ private void generateColumnArithmeticScalar(String[] tdesc) throws IOException { String operatorName = tdesc[1]; String operandType1 = tdesc[2]; String operandType2 = tdesc[3]; - String className = this.getCamelCaseType(operandType1) - + "Col" + operatorName + this.getCamelCaseType(operandType2) + "Scalar"; + String className = getCamelCaseType(operandType1) + + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar"; String returnType = getArithmeticReturnType(operandType1, operandType2); generateColumnBinaryOperatorScalar(tdesc, returnType, className); } @@ -554,19 +613,20 @@ private void generateScalarArithmeticColumn(String[] tdesc) throws IOException { String operatorName = tdesc[1]; String operandType1 = tdesc[2]; String operandType2 = tdesc[3]; - String className = this.getCamelCaseType(operandType1) - + "Scalar" + operatorName + this.getCamelCaseType(operandType2) + "Column"; + String className = getCamelCaseType(operandType1) + + "Scalar" + operatorName + getCamelCaseType(operandType2) + "Column"; String returnType = getArithmeticReturnType(operandType1, operandType2); generateScalarBinaryOperatorColumn(tdesc, returnType, className); } - private void writeFile(String outputFile, String str) throws IOException { + + static void writeFile(String outputFile, String str) throws IOException { BufferedWriter w = new BufferedWriter(new FileWriter(outputFile)); w.write(str); w.close(); } - private String readFile(String templateFile) throws IOException { + static String readFile(String templateFile) throws IOException { BufferedReader r = new BufferedReader(new FileReader(templateFile)); String line = r.readLine(); StringBuilder b = new StringBuilder(); @@ -579,7 +639,7 @@ private String readFile(String templateFile) throws IOException { return b.toString(); } - private String getCamelCaseType(String type) { + static String getCamelCaseType(String type) { if (type.equals("long")) { return "Long"; } else if (type.equals("double")) { @@ -600,9 +660,7 @@ private String getArithmeticReturnType(String operandType1, } private String getColumnVectorType(String primitiveType) { - if (primitiveType.equals("long")) { - return "LongColumnVector"; - } else if (primitiveType.equals("double")) { + if(primitiveType!=null && primitiveType.equals("double")) { return "DoubleColumnVector"; } return "LongColumnVector"; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt new file mode 100644 index 0000000..c8de5de --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import static org.junit.Assert.assertEquals; +import java.util.Random; +import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.util.VectorizedRowGroupGenUtil; +import org.junit.Test; + + +/** + * + * . + * + */ +public class { + + private static final int BATCH_SIZE = 100; + private static final long SEED = 0xfa57; + + + +} + + diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java new file mode 100644 index 0000000..34c093c --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java @@ -0,0 +1,256 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.templates; + +import java.io.IOException; +import java.util.HashMap; + +/** + * + * TestCodeGen. + * This class is mutable and maintains a hashmap of TestSuiteClassName to test cases. + * The tests cases are added over the course of vectorized expressions class generation, + * with test classes being outputted at the end. For each column vector (inputs and/or outputs) + * a matrix of pairwise covering Booleans is used to generate test cases across nulls and + * repeating dimensions. Based on the input column vector(s) nulls and repeating states + * the states of the output column vector (if there is one) is validated, along with the null + * vector. For filter operations the selection vector is validated against the generated + * data. Each template corresponds to a class representing a test suite. + */ +public class TestCodeGen { + + public enum TestSuiteClassName{ + TestColumnScalarOperationVectorExpressionEvaluation, + TestColumnScalarFilterVectorExpressionEvaluation, + TestColumnColumnOperationVectorExpressionEvaluation, + TestColumnColumnFilterVectorExpressionEvaluation, + } + + private final String testOutputDir; + private final String templateDirectory; + private final HashMap testsuites; + + public TestCodeGen(String testOutputDir, String templateDirectory) { + this.testOutputDir = testOutputDir; + this.templateDirectory = templateDirectory; + testsuites = new HashMap(); + + for(TestSuiteClassName className : TestSuiteClassName.values()) { + testsuites.put(className,new StringBuilder()); + } + + } + + public void addColumnScalarOperationTestCases(boolean op1IsCol, String vectorExpClassName, + String inputColumnVectorType, String outputColumnVectorType, String scalarType) + throws IOException { + + TestSuiteClassName template = + TestSuiteClassName.TestColumnScalarOperationVectorExpressionEvaluation; + + //Read the template into a string; + String templateFile = CodeGen.joinPath(this.templateDirectory,template.toString()+".txt"); + String templateString = removeTemplateComments(CodeGen.readFile(templateFile)); + + for(Boolean[] testMatrix :new Boolean[][]{ + // Pairwise: InitOuputColHasNulls, InitOuputColIsRepeating, ColumnHasNulls, ColumnIsRepeating + {false, true, true, true}, + {false, false, false, false}, + {true, false, true, false}, + {true, true, false, false}, + {true, false, false, true}}) { + String testCase = templateString; + testCase = testCase.replaceAll("", + "test" + + vectorExpClassName + + createNullRepeatingNameFragment("Out", testMatrix[0], testMatrix[1]) + + createNullRepeatingNameFragment("Col", testMatrix[2], testMatrix[3])); + testCase = testCase.replaceAll("", vectorExpClassName); + testCase = testCase.replaceAll("", inputColumnVectorType); + testCase = testCase.replaceAll("", outputColumnVectorType); + testCase = testCase.replaceAll("", scalarType); + testCase = testCase.replaceAll("", CodeGen.getCamelCaseType(scalarType)); + testCase = testCase.replaceAll("", testMatrix[0].toString()); + testCase = testCase.replaceAll("", testMatrix[1].toString()); + testCase = testCase.replaceAll("", testMatrix[2].toString()); + testCase = testCase.replaceAll("", testMatrix[3].toString()); + + if(op1IsCol){ + testCase = testCase.replaceAll("","0, scalarValue"); + }else{ + testCase = testCase.replaceAll("","scalarValue, 0"); + } + + testsuites.get(template).append(testCase); + } + } + + public void addColumnScalarFilterTestCases(boolean op1IsCol, String vectorExpClassName, + String inputColumnVectorType, String scalarType, String operatorSymbol) + throws IOException { + + TestSuiteClassName template = + TestSuiteClassName.TestColumnScalarFilterVectorExpressionEvaluation; + + //Read the template into a string; + String templateFile = CodeGen.joinPath(this.templateDirectory,template.toString()+".txt"); + String templateString = removeTemplateComments(CodeGen.readFile(templateFile)); + + for(Boolean[] testMatrix : new Boolean[][]{ + // Pairwise: ColumnHasNulls, ColumnIsRepeating + {true, true}, + {true, false}, + {false, false}, + {false, true}}) { + String testCase = templateString; + testCase = testCase.replaceAll("", + "test" + + vectorExpClassName + + createNullRepeatingNameFragment("Col", testMatrix[0], testMatrix[1])); + testCase = testCase.replaceAll("", vectorExpClassName); + testCase = testCase.replaceAll("", inputColumnVectorType); + testCase = testCase.replaceAll("", scalarType); + testCase = testCase.replaceAll("", CodeGen.getCamelCaseType(scalarType)); + testCase = testCase.replaceAll("", testMatrix[0].toString()); + testCase = testCase.replaceAll("", testMatrix[1].toString()); + testCase = testCase.replaceAll("", operatorSymbol); + + if(op1IsCol){ + testCase = testCase.replaceAll("","0, scalarValue"); + }else{ + testCase = testCase.replaceAll("","scalarValue, 0"); + } + + testsuites.get(template).append(testCase); + } + } + + public void addColumnColumnOperationTestCases(String vectorExpClassName, + String inputColumnVectorType1, String inputColumnVectorType2, String outputColumnVectorType) + throws IOException { + + TestSuiteClassName template= + TestSuiteClassName.TestColumnColumnOperationVectorExpressionEvaluation; + + //Read the template into a string; + String templateFile = CodeGen.joinPath(this.templateDirectory,template.toString()+".txt"); + String templateString = removeTemplateComments(CodeGen.readFile(templateFile)); + + for(Boolean[] testMatrix : new Boolean[][]{ + // Pairwise: InitOuputColHasNulls, InitOuputColIsRepeating, Column1HasNulls, + // Column1IsRepeating, Column2HasNulls, Column2IsRepeating + {true, true, false, true, true, true}, + {false, false, true, false, false, false}, + {true, false, true, false, true, true}, + {true, true, true, true, false, false}, + {false, false, false, true, true, false}, + {false, true, false, false, false, true}}) { + String testCase = templateString; + testCase = testCase.replaceAll("", + "test" + + vectorExpClassName + + createNullRepeatingNameFragment("Out", testMatrix[0], testMatrix[1]) + + createNullRepeatingNameFragment("C1", testMatrix[2], testMatrix[3]) + + createNullRepeatingNameFragment("C2", testMatrix[4], testMatrix[5])); + testCase = testCase.replaceAll("", vectorExpClassName); + testCase = testCase.replaceAll("", inputColumnVectorType1); + testCase = testCase.replaceAll("", inputColumnVectorType2); + testCase = testCase.replaceAll("", outputColumnVectorType); + testCase = testCase.replaceAll("", testMatrix[0].toString()); + testCase = testCase.replaceAll("", testMatrix[1].toString()); + testCase = testCase.replaceAll("", testMatrix[2].toString()); + testCase = testCase.replaceAll("", testMatrix[3].toString()); + testCase = testCase.replaceAll("", testMatrix[4].toString()); + testCase = testCase.replaceAll("", testMatrix[5].toString()); + + testsuites.get(template).append(testCase); + } + } + + public void addColumnColumnFilterTestCases(String vectorExpClassName, + String inputColumnVectorType1, String inputColumnVectorType2, String operatorSymbol) + throws IOException { + + TestSuiteClassName template= + TestSuiteClassName.TestColumnColumnFilterVectorExpressionEvaluation; + + //Read the template into a string; + String templateFile = CodeGen.joinPath(this.templateDirectory,template.toString()+".txt"); + String templateString = removeTemplateComments(CodeGen.readFile(templateFile)); + + for(Boolean[] testMatrix : new Boolean[][]{ + // Pairwise: Column1HasNulls, Column1IsRepeating, Column2HasNulls, Column2IsRepeating + {false, true, true, true}, + {false, false, false, false}, + {true, false, true, false}, + {true, true, false, false}, + {true, false, false, true}}) { + String testCase = templateString; + testCase = testCase.replaceAll("", + "test" + + vectorExpClassName + + createNullRepeatingNameFragment("C1", testMatrix[0], testMatrix[1]) + + createNullRepeatingNameFragment("C2", testMatrix[2], testMatrix[3])); + testCase = testCase.replaceAll("", vectorExpClassName); + testCase = testCase.replaceAll("", inputColumnVectorType1); + testCase = testCase.replaceAll("", inputColumnVectorType2); + testCase = testCase.replaceAll("", testMatrix[0].toString()); + testCase = testCase.replaceAll("", testMatrix[1].toString()); + testCase = testCase.replaceAll("", testMatrix[2].toString()); + testCase = testCase.replaceAll("", testMatrix[3].toString()); + testCase = testCase.replaceAll("", operatorSymbol); + + testsuites.get(template).append(testCase); + } + } + + public void generateTestSuites() throws IOException { + + String templateFile = CodeGen.joinPath(this.templateDirectory, "TestClass.txt"); + for(TestSuiteClassName testClass : testsuites.keySet()) { + + String templateString = CodeGen.readFile(templateFile); + templateString = templateString.replaceAll("", testClass.toString()); + templateString = templateString.replaceAll("", testsuites.get(testClass).toString()); + + String outputFile = CodeGen.joinPath(this.testOutputDir, testClass + ".java"); + + CodeGen.writeFile(outputFile, templateString); + } + } + + private static String createNullRepeatingNameFragment(String idenitfier, boolean nulls, boolean repeating) + { + if(nulls || repeating){ + if(nulls){ + idenitfier+="Nulls"; + } + if(repeating){ + idenitfier+="Repeats"; + } + return idenitfier; + } + + return ""; + } + + private static String removeTemplateComments(String templateString){ + return templateString.replaceAll("(?s)", ""); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt new file mode 100644 index 0000000..ee0b5ae --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt @@ -0,0 +1,68 @@ + + @Test + public void () { + + Random rand = new Random(SEED); + + inputColumnVector1 = + VectorizedRowGroupGenUtil.generate(, + , BATCH_SIZE, rand); + + inputColumnVector2 = + VectorizedRowGroupGenUtil.generate(, + , BATCH_SIZE, rand); + + VectorizedRowBatch rowBatch = new VectorizedRowBatch(2, BATCH_SIZE); + rowBatch.cols[0] = inputColumnVector1; + rowBatch.cols[1] = inputColumnVector2; + + vectorExpression = + new (0, 1); + + vectorExpression.evaluate(rowBatch); + + int selectedIndex = 0; + for(int i = 0; i < BATCH_SIZE; i++) { + if(!inputColumnVector1.isNull[i] && !inputColumnVector2.isNull[i]) { + if(inputColumnVector1.vector[i] inputColumnVector2.vector[i]) { + assertEquals( + "Vector index that passes filter " + + inputColumnVector1.vector[i] + "" + + inputColumnVector2.vector[i] + " is not in rowBatch selected index", + i, + rowBatch.selected[selectedIndex]); + selectedIndex++; + } + } + } + + assertEquals("Row batch size not set to number of selected rows: " + selectedIndex, + selectedIndex, rowBatch.size); + + if(selectedIndex > 0 && selectedIndex < BATCH_SIZE) { + assertEquals( + "selectedInUse should be set when > 0 and < entire batch(" + BATCH_SIZE + ") is selected: " + + selectedIndex, + true, rowBatch.selectedInUse); + }else if(selectedIndex == BATCH_SIZE) { + assertEquals( + "selectedInUse should not be set when entire batch(" + BATCH_SIZE + ") is selected: " + + selectedIndex, + false, rowBatch.selectedInUse); + } + } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt new file mode 100644 index 0000000..830c07f --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt @@ -0,0 +1,60 @@ + + @Test + public void () { + + Random rand = new Random(SEED); + + outputColumnVector = + VectorizedRowGroupGenUtil.generate(, + , BATCH_SIZE, rand); + + inputColumnVector1 = + VectorizedRowGroupGenUtil.generate(, + , BATCH_SIZE, rand); + + inputColumnVector2 = + VectorizedRowGroupGenUtil.generate(, + , BATCH_SIZE, rand); + + VectorizedRowBatch rowBatch = new VectorizedRowBatch(3, BATCH_SIZE); + rowBatch.cols[0] = inputColumnVector1; + rowBatch.cols[1] = inputColumnVector2; + rowBatch.cols[2] = outputColumnVector; + + vectorExpression = + new (0, 1, 2); + + vectorExpression.evaluate(rowBatch); + + assertEquals( + "Output column vector repeating state does not match operand columns", + inputColumnVector1.isRepeating && inputColumnVector2.isRepeating, + outputColumnVector.isRepeating); + + assertEquals( + "Output column vector no nulls state does not match operand columns", + inputColumnVector1.noNulls && inputColumnVector2.noNulls, outputColumnVector.noNulls); + + if(!rowBatch.cols[2].noNulls) { + for(int i = 0; i < BATCH_SIZE; i++) { + assertEquals("Output vector doesn't match input vectors' is null state for index", + inputColumnVector1.isNull[i] || inputColumnVector2.isNull[i], + outputColumnVector.isNull[i]); + } + } + } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt new file mode 100644 index 0000000..76b2e86 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt @@ -0,0 +1,73 @@ + + @Test + public void () { + + Random rand = new Random(SEED); + + inputColumnVector = + VectorizedRowGroupGenUtil.generate(, + , BATCH_SIZE, rand); + + VectorizedRowBatch rowBatch = new VectorizedRowBatch(1, BATCH_SIZE); + rowBatch.cols[0] = inputColumnVector; + + scalarValue=0; + do{ + scalarValue = rand.next(); + }while(scalarValue==0); + + vectorExpression = + new (); + + vectorExpression.evaluate(rowBatch); + + int selectedIndex = 0; + //check for isRepeating optimization + if(inputColumnVector.isRepeating) { + selectedIndex = !inputColumnVector.isNull[0] && inputColumnVector.vector[0] scalarValue ? BATCH_SIZE : 0; + }else { + for(int i = 0; i < BATCH_SIZE; i++) { + if(!inputColumnVector.isNull[i]) { + if(inputColumnVector.vector[i] scalarValue) { + assertEquals( + "Vector index that passes filter " + + inputColumnVector.vector[i] + "" + + scalarValue + " is not in rowBatch selected index", + i, + rowBatch.selected[selectedIndex]); + selectedIndex++; + } + } + } + } + + assertEquals("Row batch size not set to number of selected rows: " + selectedIndex, + selectedIndex, rowBatch.size); + + if(selectedIndex > 0 && selectedIndex < BATCH_SIZE) { + assertEquals( + "selectedInUse should be set when > 0 and < entire batch(" + BATCH_SIZE + ") is selected: " + + selectedIndex, + true, rowBatch.selectedInUse); + }else if(selectedIndex == BATCH_SIZE) { + assertEquals( + "selectedInUse should not be set when entire batch(" + BATCH_SIZE + ") is selected: " + + selectedIndex, + false, rowBatch.selectedInUse); + } + } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt new file mode 100644 index 0000000..4195412 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt @@ -0,0 +1,58 @@ + + @Test + public void () { + + Random rand = new Random(SEED); + + outputColumnVector = + VectorizedRowGroupGenUtil.generate(, + , BATCH_SIZE, rand); + + inputColumnVector = + VectorizedRowGroupGenUtil.generate(, + , BATCH_SIZE, rand); + + VectorizedRowBatch rowBatch = new VectorizedRowBatch(2, BATCH_SIZE); + rowBatch.cols[0] = inputColumnVector; + rowBatch.cols[1] = outputColumnVector; + + scalarValue = 0; + do{ + scalarValue = rand.next(); + }while(scalarValue == 0); + + vectorExpression = + new (, 1); + + vectorExpression.evaluate(rowBatch); + + assertEquals( + "Output column vector is repeating state does not match operand column", + inputColumnVector.isRepeating, outputColumnVector.isRepeating); + + assertEquals( + "Output column vector no nulls state does not match operand column", + inputColumnVector.noNulls, outputColumnVector.noNulls); + + if(!outputColumnVector.noNulls && !outputColumnVector.isRepeating) { + for(int i = 0; i < BATCH_SIZE; i++) { + assertEquals("Output vector doesn't match input vector's is null state for index", + inputColumnVector.isNull[i], outputColumnVector.isNull[i]); + } + } + } \ No newline at end of file diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java index 8a07567..9aad077 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java @@ -18,11 +18,18 @@ package org.apache.hadoop.hive.ql.exec.vector.util; +import java.util.Random; + +import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + public class VectorizedRowGroupGenUtil { + private static final long LONG_VECTOR_NULL_VALUE = 1; + private static final double DOUBLE_VECTOR_NULL_VALUE = Double.NaN; + public static VectorizedRowBatch getVectorizedRowBatch(int size, int numCol, int seed) { VectorizedRowBatch vrg = new VectorizedRowBatch(numCol, size); for (int j = 0; j < numCol; j++) { @@ -35,4 +42,71 @@ public static VectorizedRowBatch getVectorizedRowBatch(int size, int numCol, int vrg.size = size; return vrg; } + + public static LongColumnVector generateLongColumnVector( + boolean nulls, boolean repeating, int size, Random rand) { + LongColumnVector lcv = new LongColumnVector(size); + + lcv.noNulls = !nulls; + lcv.isRepeating = repeating; + + long repeatingValue; + do{ + repeatingValue= rand.nextLong(); + }while(repeatingValue == 0); + + int nullFrequency = generateNullFrequency(rand); + + for(int i = 0; i < size; i++) { + if(nulls && (repeating || i % nullFrequency == 0)) { + lcv.isNull[i] = true; + lcv.vector[i] = LONG_VECTOR_NULL_VALUE; + + }else { + lcv.isNull[i] = false; + lcv.vector[i] = repeating ? repeatingValue : rand.nextLong(); + if(lcv.vector[i] == 0) { + i--; + } + } + } + return lcv; + } + + public static DoubleColumnVector generateDoubleColumnVector( + boolean nulls, boolean repeating, int size, Random rand) + { + DoubleColumnVector dcv = new DoubleColumnVector(size); + + dcv.noNulls = !nulls; + dcv.isRepeating = repeating; + + double repeatingValue; + do{ + repeatingValue= rand.nextDouble(); + }while(repeatingValue == 0); + + int nullFrequency = generateNullFrequency(rand); + + for(int i = 0; i < size; i++) { + if(nulls && (repeating || i % nullFrequency == 0)) { + dcv.isNull[i] = true; + dcv.vector[i] = DOUBLE_VECTOR_NULL_VALUE; + + }else { + dcv.isNull[i] = false; + dcv.vector[i] = repeating ? repeatingValue : rand.nextDouble(); + + if(dcv.vector[i] == 0) { + i--; + } + } + } + return dcv; + } + + private static int generateNullFrequency(Random rand) { + return 60 + rand.nextInt(20); + } + } -- 1.8.0.msysgit.0