From e900ce311f5a4c25cdfffb1a1b620194429438d6 Mon Sep 17 00:00:00 2001 From: Tony Murphy Date: Thu, 23 May 2013 12:09:09 -0700 Subject: [PATCH] HIVE-4597 --- .../ql/exec/vector/expressions/gen/CodeGen.java | 710 +++++++++++++++++++++ .../expressions/gen/ColumnArithmeticColumn.txt | 178 ++++++ .../expressions/gen/ColumnArithmeticScalar.txt | 104 +++ .../vector/expressions/gen/ColumnCompareScalar.txt | 114 ++++ .../vector/expressions/gen/ColumnUnaryMinus.txt | 101 +++ .../expressions/gen/FilterColumnCompareColumn.txt | 227 +++++++ .../expressions/gen/FilterColumnCompareScalar.txt | 132 ++++ .../gen/FilterStringColumnCompareColumn.txt | 455 +++++++++++++ .../gen/FilterStringColumnCompareScalar.txt | 139 ++++ .../expressions/gen/ScalarArithmeticColumn.txt | 123 ++++ .../ql/exec/vector/expressions/gen/TestClass.txt | 44 ++ .../exec/vector/expressions/gen/TestCodeGen.java | 256 ++++++++ ...olumnColumnFilterVectorExpressionEvaluation.txt | 69 ++ ...mnColumnOperationVectorExpressionEvaluation.txt | 64 ++ ...olumnScalarFilterVectorExpressionEvaluation.txt | 76 +++ ...mnScalarOperationVectorExpressionEvaluation.txt | 59 ++ .../exec/vector/expressions/gen/VectorUDAFAvg.txt | 440 +++++++++++++ .../vector/expressions/gen/VectorUDAFCount.txt | 262 ++++++++ .../vector/expressions/gen/VectorUDAFMinMax.txt | 415 ++++++++++++ .../exec/vector/expressions/gen/VectorUDAFSum.txt | 409 ++++++++++++ .../exec/vector/expressions/gen/VectorUDAFVar.txt | 487 ++++++++++++++ .../exec/vector/expressions/templates/CodeGen.java | 697 -------------------- .../templates/ColumnArithmeticColumn.txt | 178 ------ .../templates/ColumnArithmeticScalar.txt | 104 --- .../expressions/templates/ColumnCompareScalar.txt | 114 ---- .../expressions/templates/ColumnUnaryMinus.txt | 101 --- .../templates/FilterColumnCompareColumn.txt | 227 ------- .../templates/FilterColumnCompareScalar.txt | 132 ---- .../templates/FilterStringColumnCompareColumn.txt | 455 ------------- .../templates/FilterStringColumnCompareScalar.txt | 139 ---- .../templates/ScalarArithmeticColumn.txt | 123 ---- .../vector/expressions/templates/TestClass.txt | 44 -- .../vector/expressions/templates/TestCodeGen.java | 256 -------- ...olumnColumnFilterVectorExpressionEvaluation.txt | 69 -- ...mnColumnOperationVectorExpressionEvaluation.txt | 64 -- ...olumnScalarFilterVectorExpressionEvaluation.txt | 76 --- ...mnScalarOperationVectorExpressionEvaluation.txt | 59 -- .../vector/expressions/templates/VectorUDAFAvg.txt | 440 ------------- .../expressions/templates/VectorUDAFCount.txt | 262 -------- .../expressions/templates/VectorUDAFMinMax.txt | 415 ------------ .../vector/expressions/templates/VectorUDAFSum.txt | 409 ------------ .../vector/expressions/templates/VectorUDAFVar.txt | 487 -------------- 42 files changed, 4864 insertions(+), 4851 deletions(-) create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/CodeGen.java create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ColumnArithmeticColumn.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ColumnArithmeticScalar.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ColumnCompareScalar.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ColumnUnaryMinus.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterColumnCompareColumn.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterColumnCompareScalar.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColumnCompareColumn.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColumnCompareScalar.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ScalarArithmeticColumn.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestClass.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestCodeGen.java create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnColumnFilterVectorExpressionEvaluation.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnColumnOperationVectorExpressionEvaluation.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnScalarFilterVectorExpressionEvaluation.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnScalarOperationVectorExpressionEvaluation.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFAvg.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFCount.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFMinMax.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFSum.txt create mode 100644 ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFVar.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ColumnArithmeticColumn.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ColumnArithmeticScalar.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ColumnCompareScalar.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ColumnUnaryMinus.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterColumnCompareColumn.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterColumnCompareScalar.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterStringColumnCompareColumn.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterStringColumnCompareScalar.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ScalarArithmeticColumn.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFAvg.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFCount.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMax.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFSum.txt delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/CodeGen.java ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/CodeGen.java new file mode 100644 index 0000000..0f1d816 --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/CodeGen.java @@ -0,0 +1,710 @@ +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; + +public class CodeGen { + + static String [][] templateExpansions = + { + {"ColumnArithmeticScalar", "Add", "long", "long", "+"}, + {"ColumnArithmeticScalar", "Subtract", "long", "long", "-"}, + {"ColumnArithmeticScalar", "Multiply", "long", "long", "*"}, + {"ColumnArithmeticScalar", "Modulo", "long", "long", "%"}, + + {"ColumnArithmeticScalar", "Add", "long", "double", "+"}, + {"ColumnArithmeticScalar", "Subtract", "long", "double", "-"}, + {"ColumnArithmeticScalar", "Multiply", "long", "double", "*"}, + {"ColumnArithmeticScalar", "Divide", "long", "double", "/"}, + {"ColumnArithmeticScalar", "Modulo", "long", "double", "%"}, + + {"ColumnArithmeticScalar", "Add", "double", "long", "+"}, + {"ColumnArithmeticScalar", "Subtract", "double", "long", "-"}, + {"ColumnArithmeticScalar", "Multiply", "double", "long", "*"}, + {"ColumnArithmeticScalar", "Divide", "double", "long", "/"}, + {"ColumnArithmeticScalar", "Modulo", "double", "long", "%"}, + + {"ColumnArithmeticScalar", "Add", "double", "double", "+"}, + {"ColumnArithmeticScalar", "Subtract", "double", "double", "-"}, + {"ColumnArithmeticScalar", "Multiply", "double", "double", "*"}, + {"ColumnArithmeticScalar", "Divide", "double", "double", "/"}, + {"ColumnArithmeticScalar", "Modulo", "double", "double", "%"}, + + {"ScalarArithmeticColumn", "Add", "long", "long", "+"}, + {"ScalarArithmeticColumn", "Subtract", "long", "long", "-"}, + {"ScalarArithmeticColumn", "Multiply", "long", "long", "*"}, + {"ScalarArithmeticColumn", "Modulo", "long", "long", "%"}, + + {"ScalarArithmeticColumn", "Add", "long", "double", "+"}, + {"ScalarArithmeticColumn", "Subtract", "long", "double", "-"}, + {"ScalarArithmeticColumn", "Multiply", "long", "double", "*"}, + {"ScalarArithmeticColumn", "Divide", "long", "double", "/"}, + {"ScalarArithmeticColumn", "Modulo", "long", "double", "%"}, + + {"ScalarArithmeticColumn", "Add", "double", "long", "+"}, + {"ScalarArithmeticColumn", "Subtract", "double", "long", "-"}, + {"ScalarArithmeticColumn", "Multiply", "double", "long", "*"}, + {"ScalarArithmeticColumn", "Divide", "double", "long", "/"}, + {"ScalarArithmeticColumn", "Modulo", "double", "long", "%"}, + + {"ScalarArithmeticColumn", "Add", "double", "double", "+"}, + {"ScalarArithmeticColumn", "Subtract", "double", "double", "-"}, + {"ScalarArithmeticColumn", "Multiply", "double", "double", "*"}, + {"ScalarArithmeticColumn", "Divide", "double", "double", "/"}, + {"ScalarArithmeticColumn", "Modulo", "double", "double", "%"}, + + {"ColumnArithmeticColumn", "Add", "long", "long", "+"}, + {"ColumnArithmeticColumn", "Subtract", "long", "long", "-"}, + {"ColumnArithmeticColumn", "Multiply", "long", "long", "*"}, + {"ColumnArithmeticColumn", "Modulo", "long", "long", "%"}, + + {"ColumnArithmeticColumn", "Add", "long", "double", "+"}, + {"ColumnArithmeticColumn", "Subtract", "long", "double", "-"}, + {"ColumnArithmeticColumn", "Multiply", "long", "double", "*"}, + {"ColumnArithmeticColumn", "Divide", "long", "double", "/"}, + {"ColumnArithmeticColumn", "Modulo", "long", "double", "%"}, + + {"ColumnArithmeticColumn", "Add", "double", "long", "+"}, + {"ColumnArithmeticColumn", "Subtract", "double", "long", "-"}, + {"ColumnArithmeticColumn", "Multiply", "double", "long", "*"}, + {"ColumnArithmeticColumn", "Divide", "double", "long", "/"}, + {"ColumnArithmeticColumn", "Modulo", "double", "long", "%"}, + + {"ColumnArithmeticColumn", "Add", "double", "double", "+"}, + {"ColumnArithmeticColumn", "Subtract", "double", "double", "-"}, + {"ColumnArithmeticColumn", "Multiply", "double", "double", "*"}, + {"ColumnArithmeticColumn", "Divide", "double", "double", "/"}, + {"ColumnArithmeticColumn", "Modulo", "double", "double", "%"}, + + {"ColumnCompareScalar", "Equal", "long", "double", "=="}, + {"ColumnCompareScalar", "Equal", "double", "double", "=="}, + {"ColumnCompareScalar", "NotEqual", "long", "double", "!="}, + {"ColumnCompareScalar", "NotEqual", "double", "double", "!="}, + {"ColumnCompareScalar", "Less", "long", "double", "<"}, + {"ColumnCompareScalar", "Less", "double", "double", "<"}, + {"ColumnCompareScalar", "LessEqual", "long", "double", "<="}, + {"ColumnCompareScalar", "LessEqual", "double", "double", "<="}, + {"ColumnCompareScalar", "Greater", "long", "double", ">"}, + {"ColumnCompareScalar", "Greater", "double", "double", ">"}, + {"ColumnCompareScalar", "GreaterEqual", "long", "double", ">="}, + {"ColumnCompareScalar", "GreaterEqual", "double", "double", ">="}, + + {"FilterColumnCompareScalar", "Equal", "long", "double", "=="}, + {"FilterColumnCompareScalar", "Equal", "double", "double", "=="}, + {"FilterColumnCompareScalar", "NotEqual", "long", "double", "!="}, + {"FilterColumnCompareScalar", "NotEqual", "double", "double", "!="}, + {"FilterColumnCompareScalar", "Less", "long", "double", "<"}, + {"FilterColumnCompareScalar", "Less", "double", "double", "<"}, + {"FilterColumnCompareScalar", "LessEqual", "long", "double", "<="}, + {"FilterColumnCompareScalar", "LessEqual", "double", "double", "<="}, + {"FilterColumnCompareScalar", "Greater", "long", "double", ">"}, + {"FilterColumnCompareScalar", "Greater", "double", "double", ">"}, + {"FilterColumnCompareScalar", "GreaterEqual", "long", "double", ">="}, + {"FilterColumnCompareScalar", "GreaterEqual", "double", "double", ">="}, + + {"FilterColumnCompareScalar", "Equal", "long", "long", "=="}, + {"FilterColumnCompareScalar", "Equal", "double", "long", "=="}, + {"FilterColumnCompareScalar", "NotEqual", "long", "long", "!="}, + {"FilterColumnCompareScalar", "NotEqual", "double", "long", "!="}, + {"FilterColumnCompareScalar", "Less", "long", "long", "<"}, + {"FilterColumnCompareScalar", "Less", "double", "long", "<"}, + {"FilterColumnCompareScalar", "LessEqual", "long", "long", "<="}, + {"FilterColumnCompareScalar", "LessEqual", "double", "long", "<="}, + {"FilterColumnCompareScalar", "Greater", "long", "long", ">"}, + {"FilterColumnCompareScalar", "Greater", "double", "long", ">"}, + {"FilterColumnCompareScalar", "GreaterEqual", "long", "long", ">="}, + {"FilterColumnCompareScalar", "GreaterEqual", "double", "long", ">="}, + + {"FilterScalarCompareColumn", "Equal", "long", "double", "=="}, + {"FilterScalarCompareColumn", "Equal", "double", "double", "=="}, + {"FilterScalarCompareColumn", "NotEqual", "long", "double", "!="}, + {"FilterScalarCompareColumn", "NotEqual", "double", "double", "!="}, + {"FilterScalarCompareColumn", "Less", "long", "double", "<"}, + {"FilterScalarCompareColumn", "Less", "double", "double", "<"}, + {"FilterScalarCompareColumn", "LessEqual", "long", "double", "<="}, + {"FilterScalarCompareColumn", "LessEqual", "double", "double", "<="}, + {"FilterScalarCompareColumn", "Greater", "long", "double", ">"}, + {"FilterScalarCompareColumn", "Greater", "double", "double", ">"}, + {"FilterScalarCompareColumn", "GreaterEqual", "long", "double", ">="}, + {"FilterScalarCompareColumn", "GreaterEqual", "double", "double", ">="}, + + {"FilterScalarCompareColumn", "Equal", "long", "long", "=="}, + {"FilterScalarCompareColumn", "Equal", "double", "long", "=="}, + {"FilterScalarCompareColumn", "NotEqual", "long", "long", "!="}, + {"FilterScalarCompareColumn", "NotEqual", "double", "long", "!="}, + {"FilterScalarCompareColumn", "Less", "long", "long", "<"}, + {"FilterScalarCompareColumn", "Less", "double", "long", "<"}, + {"FilterScalarCompareColumn", "LessEqual", "long", "long", "<="}, + {"FilterScalarCompareColumn", "LessEqual", "double", "long", "<="}, + {"FilterScalarCompareColumn", "Greater", "long", "long", ">"}, + {"FilterScalarCompareColumn", "Greater", "double", "long", ">"}, + {"FilterScalarCompareColumn", "GreaterEqual", "long", "long", ">="}, + {"FilterScalarCompareColumn", "GreaterEqual", "double", "long", ">="}, + + {"FilterStringColumnCompareScalar", "Equal", "=="}, + {"FilterStringColumnCompareScalar", "NotEqual", "!="}, + {"FilterStringColumnCompareScalar", "Less", "<"}, + {"FilterStringColumnCompareScalar", "LessEqual", "<="}, + {"FilterStringColumnCompareScalar", "Greater", ">"}, + {"FilterStringColumnCompareScalar", "GreaterEqual", ">="}, + + {"FilterStringColumnCompareColumn", "Equal", "=="}, + {"FilterStringColumnCompareColumn", "NotEqual", "!="}, + {"FilterStringColumnCompareColumn", "Less", "<"}, + {"FilterStringColumnCompareColumn", "LessEqual", "<="}, + {"FilterStringColumnCompareColumn", "Greater", ">"}, + {"FilterStringColumnCompareColumn", "GreaterEqual", ">="}, + + {"FilterColumnCompareColumn", "Equal", "long", "double", "=="}, + {"FilterColumnCompareColumn", "Equal", "double", "double", "=="}, + {"FilterColumnCompareColumn", "NotEqual", "long", "double", "!="}, + {"FilterColumnCompareColumn", "NotEqual", "double", "double", "!="}, + {"FilterColumnCompareColumn", "Less", "long", "double", "<"}, + {"FilterColumnCompareColumn", "Less", "double", "double", "<"}, + {"FilterColumnCompareColumn", "LessEqual", "long", "double", "<="}, + {"FilterColumnCompareColumn", "LessEqual", "double", "double", "<="}, + {"FilterColumnCompareColumn", "Greater", "long", "double", ">"}, + {"FilterColumnCompareColumn", "Greater", "double", "double", ">"}, + {"FilterColumnCompareColumn", "GreaterEqual", "long", "double", ">="}, + {"FilterColumnCompareColumn", "GreaterEqual", "double", "double", ">="}, + + {"FilterColumnCompareColumn", "Equal", "long", "long", "=="}, + {"FilterColumnCompareColumn", "Equal", "double", "long", "=="}, + {"FilterColumnCompareColumn", "NotEqual", "long", "long", "!="}, + {"FilterColumnCompareColumn", "NotEqual", "double", "long", "!="}, + {"FilterColumnCompareColumn", "Less", "long", "long", "<"}, + {"FilterColumnCompareColumn", "Less", "double", "long", "<"}, + {"FilterColumnCompareColumn", "LessEqual", "long", "long", "<="}, + {"FilterColumnCompareColumn", "LessEqual", "double", "long", "<="}, + {"FilterColumnCompareColumn", "Greater", "long", "long", ">"}, + {"FilterColumnCompareColumn", "Greater", "double", "long", ">"}, + {"FilterColumnCompareColumn", "GreaterEqual", "long", "long", ">="}, + {"FilterColumnCompareColumn", "GreaterEqual", "double", "long", ">="}, + + {"ColumnUnaryMinus", "long"}, + {"ColumnUnaryMinus", "double"}, + + //template, , , , , + {"VectorUDAFMinMax", "VectorUDAFMinLong", "long", "<", "min", "_FUNC_(expr) - Returns the minimum value of expr (vectorized, type: long)"}, + {"VectorUDAFMinMax", "VectorUDAFMinDouble", "double", "<", "min", "_FUNC_(expr) - Returns the minimum value of expr (vectorized, type: double)"}, + {"VectorUDAFMinMax", "VectorUDAFMaxLong", "long", ">", "max", "_FUNC_(expr) - Returns the maximum value of expr (vectorized, type: long)"}, + {"VectorUDAFMinMax", "VectorUDAFMaxDouble", "double", ">", "max", "_FUNC_(expr) - Returns the maximum value of expr (vectorized, type: double)"}, + + //template, , + {"VectorUDAFCount", "VectorUDAFCountLong", "long"}, + {"VectorUDAFCount", "VectorUDAFCountDouble", "double"}, + + //template, , + {"VectorUDAFSum", "VectorUDAFSumLong", "long"}, + {"VectorUDAFSum", "VectorUDAFSumDouble", "double"}, + {"VectorUDAFAvg", "VectorUDAFAvgLong", "long"}, + {"VectorUDAFAvg", "VectorUDAFAvgDouble", "double"}, + + //template, , , , , + {"VectorUDAFVar", "VectorUDAFVarPopLong", "long", "myagg.variance / myagg.count", "variance, var_pop", "_FUNC_(x) - Returns the variance of a set of numbers (vectorized, long)"}, + {"VectorUDAFVar", "VectorUDAFVarPopDouble", "double", "myagg.variance / myagg.count", "variance, var_pop", "_FUNC_(x) - Returns the variance of a set of numbers (vectorized, double)"}, + {"VectorUDAFVar", "VectorUDAFVarSampLong", "long", "myagg.variance / (myagg.count-1.0)", "var_samp", "_FUNC_(x) - Returns the sample variance of a set of numbers (vectorized, long)"}, + {"VectorUDAFVar", "VectorUDAFVarSampDouble", "double", "myagg.variance / (myagg.count-1.0)", "var_samp", "_FUNC_(x) - Returns the sample variance of a set of numbers (vectorized, double)"}, + {"VectorUDAFVar", "VectorUDAFStdPopLong", "long", "Math.sqrt(myagg.variance / (myagg.count))", "std,stddev,stddev_pop", "_FUNC_(x) - Returns the standard deviation of a set of numbers (vectorized, long)"}, + {"VectorUDAFVar", "VectorUDAFStdPopDouble", "double", "Math.sqrt(myagg.variance / (myagg.count))", "std,stddev,stddev_pop", "_FUNC_(x) - Returns the standard deviation of a set of numbers (vectorized, double)"}, + {"VectorUDAFVar", "VectorUDAFStdSampLong", "long", "Math.sqrt(myagg.variance / (myagg.count-1.0))", "stddev_samp", "_FUNC_(x) - Returns the sample standard deviation of a set of numbers (vectorized, long)"}, + {"VectorUDAFVar", "VectorUDAFStdSampDouble", "double", "Math.sqrt(myagg.variance / (myagg.count-1.0))", "stddev_samp", "_FUNC_(x) - Returns the sample standard deviation of a set of numbers (vectorized, double)"}, + + }; + private static String[] relativePackagePathParts = new String[]{"org", "apache", "hadoop", "hive", "ql", "exec", "vector", "expressions", "gen"}; + private static String[] relativeSrcPathParts = new String[]{"..", "..", "java"}; + private static String[] relativeTestPathParts = new String[]{"..", "..", "test"}; + + private final String templateDirectory; + private final String outputDirectory; + private final TestCodeGen testCodeGen; + + static String joinPath(String...parts) { + String path = parts[0]; + for (int i=1; i < parts.length; ++i) { + path += File.separatorChar + parts[i]; + } + return path; + } + + public CodeGen() { + + String relativePackagePath = joinPath(relativePackagePathParts); + templateDirectory = joinPath(System.getProperty("user.dir"), relativePackagePath); + + String relativeSrcPath = joinPath(relativeSrcPathParts); + outputDirectory = joinPath(System.getProperty("user.dir"), relativeSrcPath, relativePackagePath); + + String relativeTestPath = joinPath(relativeTestPathParts); + String testOutputDirectory = joinPath(System.getProperty("user.dir"), relativeTestPath, relativePackagePath); + testCodeGen = new TestCodeGen(testOutputDirectory, templateDirectory); + + System.out.println("Template Directory: " + templateDirectory); + System.out.println("Output Directory: " + outputDirectory); + System.out.println("Test Output Directory: " + testOutputDirectory); + } + + public CodeGen(String templateDirectory, String outputDirectory, String testOutputDirectory) { + this.templateDirectory = templateDirectory; + this.outputDirectory = outputDirectory; + testCodeGen = new TestCodeGen(testOutputDirectory, templateDirectory); + } + + /** + * @param args + * @throws Exception + */ + public static void main(String[] args) throws Exception { + CodeGen gen; + if (args == null || args.length==0) { + gen = new CodeGen(); + } else if (args.length==3) { + gen = new CodeGen(args[0], args[1], args[2]); + }else{ + System.out.println("args: "); + return; + } + + gen.generate(); + } + + private void generate() throws Exception { + + for (String [] tdesc : templateExpansions) { + if (tdesc[0].equals("ColumnArithmeticScalar")) { + generateColumnArithmeticScalar(tdesc); + } else if (tdesc[0].equals("ColumnCompareScalar")) { + generateColumnCompareScalar(tdesc); + } else if (tdesc[0].equals("FilterColumnCompareScalar")) { + generateFilterColumnCompareScalar(tdesc); + } else if (tdesc[0].equals("ScalarArithmeticColumn")) { + generateScalarArithmeticColumn(tdesc); + } else if (tdesc[0].equals("FilterColumnCompareColumn")) { + generateFilterColumnCompareColumn(tdesc); + } else if (tdesc[0].equals("ColumnArithmeticColumn")) { + generateColumnArithmeticColumn(tdesc); + } else if (tdesc[0].equals("ColumnUnaryMinus")) { + generateColumnUnaryMinus(tdesc); + } else if (tdesc[0].equals("VectorUDAFCount")) { + generateVectorUDAFCount(tdesc); + } else if (tdesc[0].equals("VectorUDAFMinMax")) { + generateVectorUDAFMinMax(tdesc); + } else if (tdesc[0].equals("VectorUDAFSum")) { + generateVectorUDAFSum(tdesc); + } else if (tdesc[0].equals("VectorUDAFAvg")) { + generateVectorUDAFAvg(tdesc); + } else if (tdesc[0].equals("VectorUDAFVar")) { + generateVectorUDAFVar(tdesc); + } else if (tdesc[0].equals("FilterStringColumnCompareScalar")) { + generateFilterStringColumnCompareScalar(tdesc); + } else if (tdesc[0].equals("FilterStringColumnCompareColumn")) { + generateFilterStringColumnCompareColumn(tdesc); + } else { + continue; + } + } + testCodeGen.generateTestSuites(); + } + + private void generateVectorUDAFMinMax(String[] tdesc) throws Exception { + String className = tdesc[1]; + String valueType = tdesc[2]; + String operatorSymbol = tdesc[3]; + String descName = tdesc[4]; + String descValue = tdesc[5]; + String columnType = getColumnVectorType(valueType); + String writableType = getOutputWritableType(valueType); + String inspectorType = getOutputObjectInspector(valueType); + + String outputFile = joinPath(this.outputDirectory, className + ".java"); + String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); + + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", valueType); + templateString = templateString.replaceAll("", operatorSymbol); + templateString = templateString.replaceAll("", columnType); + templateString = templateString.replaceAll("", descName); + templateString = templateString.replaceAll("", descValue); + templateString = templateString.replaceAll("", writableType); + templateString = templateString.replaceAll("", inspectorType); + writeFile(outputFile, templateString); + + } + + private void generateVectorUDAFCount(String[] tdesc) throws IOException { + String className = tdesc[1]; + String valueType = tdesc[2]; + String columnType = getColumnVectorType(valueType); + + String outputFile = joinPath(this.outputDirectory, className + ".java"); + String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); + + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", valueType); + templateString = templateString.replaceAll("", columnType); + writeFile(outputFile, templateString); + } + + private void generateVectorUDAFSum(String[] tdesc) throws Exception { + //template, , , , + String className = tdesc[1]; + String valueType = tdesc[2]; + String columnType = getColumnVectorType(valueType); + String writableType = getOutputWritableType(valueType); + String inspectorType = getOutputObjectInspector(valueType); + + String outputFile = joinPath(this.outputDirectory, className + ".java"); + String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); + + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", valueType); + templateString = templateString.replaceAll("", columnType); + templateString = templateString.replaceAll("", writableType); + templateString = templateString.replaceAll("", inspectorType); + writeFile(outputFile, templateString); + } + + private void generateVectorUDAFAvg(String[] tdesc) throws IOException { + String className = tdesc[1]; + String valueType = tdesc[2]; + String columnType = getColumnVectorType(valueType); + + String outputFile = joinPath(this.outputDirectory, className + ".java"); + String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); + + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", valueType); + templateString = templateString.replaceAll("", columnType); + writeFile(outputFile, templateString); + } + + private void generateVectorUDAFVar(String[] tdesc) throws IOException { + String className = tdesc[1]; + String valueType = tdesc[2]; + String varianceFormula = tdesc[3]; + String descriptionName = tdesc[4]; + String descriptionValue = tdesc[5]; + String columnType = getColumnVectorType(valueType); + + String outputFile = joinPath(this.outputDirectory, className + ".java"); + String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); + + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", valueType); + templateString = templateString.replaceAll("", columnType); + templateString = templateString.replaceAll("", varianceFormula); + templateString = templateString.replaceAll("", descriptionName); + templateString = templateString.replaceAll("", descriptionValue); + writeFile(outputFile, templateString); + } + + + private void generateFilterStringColumnCompareScalar(String[] tdesc) throws IOException { + String operatorName = tdesc[1]; + String className = "FilterStringCol" + operatorName + "StringScalar"; + generateFilterStringColumnCompareScalar(tdesc,className); + } + + private void generateFilterStringColumnCompareColumn(String[] tdesc) throws IOException { + String operatorName = tdesc[1]; + String className = "FilterStringCol" + operatorName + "StringCol"; + generateFilterStringColumnCompareScalar(tdesc,className); + } + + private void generateFilterStringColumnCompareScalar(String[] tdesc, String className) throws IOException { + String operatorSymbol = tdesc[2]; + String outputFile = joinPath(this.outputDirectory, className + ".java"); + // Read the template into a string; + String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); + String templateString = readFile(templateFile); + // Expand, and write result + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", operatorSymbol); + writeFile(outputFile, templateString); + } + + private void generateFilterColumnCompareColumn(String[] tdesc) throws IOException { + //The variables are all same as ColumnCompareScalar except that + //this template doesn't need a return type. Pass anything as return type. + String operatorName = tdesc[1]; + String operandType1 = tdesc[2]; + String operandType2 = tdesc[3]; + String className = "Filter" + getCamelCaseType(operandType1) + + "Col" + operatorName + getCamelCaseType(operandType2) + "Column"; + generateColumnBinaryOperatorColumn(tdesc, null, className); + } + + private void generateColumnUnaryMinus(String[] tdesc) throws IOException { + String operandType = tdesc[1]; + String inputColumnVectorType = this.getColumnVectorType(operandType); + String outputColumnVectorType = inputColumnVectorType; + String returnType = operandType; + String className = getCamelCaseType(operandType) + "ColUnaryMinus"; + String outputFile = joinPath(this.outputDirectory, className + ".java"); + String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); + String templateString = readFile(templateFile); + // Expand, and write result + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", inputColumnVectorType); + templateString = templateString.replaceAll("", outputColumnVectorType); + templateString = templateString.replaceAll("", operandType); + templateString = templateString.replaceAll("", returnType); + writeFile(outputFile, templateString); + } + + private void generateColumnArithmeticColumn(String [] tdesc) throws IOException { + String operatorName = tdesc[1]; + String operandType1 = tdesc[2]; + String operandType2 = tdesc[3]; + String className = getCamelCaseType(operandType1) + + "Col" + operatorName + getCamelCaseType(operandType2) + "Column"; + String returnType = getArithmeticReturnType(operandType1, operandType2); + generateColumnBinaryOperatorColumn(tdesc, returnType, className); + } + + private void generateFilterColumnCompareScalar(String[] tdesc) throws IOException { + //The variables are all same as ColumnCompareScalar except that + //this template doesn't need a return type. Pass anything as return type. + String operatorName = tdesc[1]; + String operandType1 = tdesc[2]; + String operandType2 = tdesc[3]; + String className = "Filter" + getCamelCaseType(operandType1) + + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar"; + generateColumnBinaryOperatorScalar(tdesc, null, className); + } + + private void generateFilterScalarCompareColumn(String[] tdesc) throws IOException { + //this template doesn't need a return type. Pass anything as return type. + String operatorName = tdesc[1]; + String operandType1 = tdesc[2]; + String operandType2 = tdesc[3]; + String className = "Filter" + getCamelCaseType(operandType1) + + "Scalar" + operatorName + getCamelCaseType(operandType2) + "Column"; + generateScalarBinaryOperatorColumn(tdesc, null, className); + } + + private void generateColumnCompareScalar(String[] tdesc) throws IOException { + String operatorName = tdesc[1]; + String operandType1 = tdesc[2]; + String operandType2 = tdesc[3]; + String returnType = "long"; + String className = getCamelCaseType(operandType1) + + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar"; + generateColumnBinaryOperatorScalar(tdesc, returnType, className); + } + + private void generateColumnBinaryOperatorColumn(String[] tdesc, String returnType, + String className) throws IOException { + String operandType1 = tdesc[2]; + String operandType2 = tdesc[3]; + String outputColumnVectorType = this.getColumnVectorType(returnType); + String inputColumnVectorType1 = this.getColumnVectorType(operandType1); + String inputColumnVectorType2 = this.getColumnVectorType(operandType2); + String operatorSymbol = tdesc[4]; + String outputFile = joinPath(this.outputDirectory, className + ".java"); + + //Read the template into a string; + String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", inputColumnVectorType1); + templateString = templateString.replaceAll("", inputColumnVectorType2); + templateString = templateString.replaceAll("", outputColumnVectorType); + templateString = templateString.replaceAll("", operatorSymbol); + templateString = templateString.replaceAll("", operandType1); + templateString = templateString.replaceAll("", operandType2); + templateString = templateString.replaceAll("", returnType); + writeFile(outputFile, templateString); + + if(returnType==null){ + testCodeGen.addColumnColumnFilterTestCases( + className, + inputColumnVectorType1, + inputColumnVectorType2, + operatorSymbol); + }else{ + testCodeGen.addColumnColumnOperationTestCases( + className, + inputColumnVectorType1, + inputColumnVectorType2, + outputColumnVectorType); + } + } + + private void generateColumnBinaryOperatorScalar(String[] tdesc, String returnType, + String className) throws IOException { + String operandType1 = tdesc[2]; + String operandType2 = tdesc[3]; + String outputColumnVectorType = this.getColumnVectorType(returnType); + String inputColumnVectorType = this.getColumnVectorType(operandType1); + String operatorSymbol = tdesc[4]; + String outputFile = joinPath(this.outputDirectory, className + ".java"); + + //Read the template into a string; + String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", inputColumnVectorType); + templateString = templateString.replaceAll("", outputColumnVectorType); + templateString = templateString.replaceAll("", operatorSymbol); + templateString = templateString.replaceAll("", operandType1); + templateString = templateString.replaceAll("", operandType2); + templateString = templateString.replaceAll("", returnType); + writeFile(outputFile, templateString); + + if(returnType==null) + { + testCodeGen.addColumnScalarFilterTestCases( + true, + className, + inputColumnVectorType, + operandType2, + operatorSymbol); + }else + { + testCodeGen.addColumnScalarOperationTestCases( + true, + className, + inputColumnVectorType, + outputColumnVectorType, + operandType2); + } + + } + + private void generateScalarBinaryOperatorColumn(String[] tdesc, String returnType, + String className) throws IOException { + String operandType1 = tdesc[2]; + String operandType2 = tdesc[3]; + String outputColumnVectorType = this.getColumnVectorType(returnType); + String inputColumnVectorType = this.getColumnVectorType(operandType2); + String operatorSymbol = tdesc[4]; + String outputFile = joinPath(this.outputDirectory, className + ".java"); + + //Read the template into a string; + String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", inputColumnVectorType); + templateString = templateString.replaceAll("", outputColumnVectorType); + templateString = templateString.replaceAll("", operatorSymbol); + templateString = templateString.replaceAll("", operandType1); + templateString = templateString.replaceAll("", operandType2); + templateString = templateString.replaceAll("", returnType); + writeFile(outputFile, templateString); + + if(returnType==null) + { + testCodeGen.addColumnScalarFilterTestCases( + false, + className, + inputColumnVectorType, + operandType1, + operatorSymbol); + }else + { + testCodeGen.addColumnScalarOperationTestCases( + false, + className, + inputColumnVectorType, + outputColumnVectorType, + operandType1); + } + } + + //Binary arithmetic operator + private void generateColumnArithmeticScalar(String[] tdesc) throws IOException { + String operatorName = tdesc[1]; + String operandType1 = tdesc[2]; + String operandType2 = tdesc[3]; + String className = getCamelCaseType(operandType1) + + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar"; + String returnType = getArithmeticReturnType(operandType1, operandType2); + generateColumnBinaryOperatorScalar(tdesc, returnType, className); + } + + private void generateScalarArithmeticColumn(String[] tdesc) throws IOException { + String operatorName = tdesc[1]; + String operandType1 = tdesc[2]; + String operandType2 = tdesc[3]; + String className = getCamelCaseType(operandType1) + + "Scalar" + operatorName + getCamelCaseType(operandType2) + "Column"; + String returnType = getArithmeticReturnType(operandType1, operandType2); + generateScalarBinaryOperatorColumn(tdesc, returnType, className); + } + + + static void writeFile(String outputFile, String str) throws IOException { + BufferedWriter w = new BufferedWriter(new FileWriter(outputFile)); + w.write(str); + w.close(); + } + + static String readFile(String templateFile) throws IOException { + BufferedReader r = new BufferedReader(new FileReader(templateFile)); + String line = r.readLine(); + StringBuilder b = new StringBuilder(); + while (line != null) { + b.append(line); + b.append("\n"); + line = r.readLine(); + } + r.close(); + return b.toString(); + } + + static String getCamelCaseType(String type) { + if (type.equals("long")) { + return "Long"; + } else if (type.equals("double")) { + return "Double"; + } else { + return type; + } + } + + private String getArithmeticReturnType(String operandType1, + String operandType2) { + if (operandType1.equals("double") || + operandType2.equals("double")) { + return "double"; + } else { + return "long"; + } + } + + private String getColumnVectorType(String primitiveType) { + if(primitiveType!=null && primitiveType.equals("double")) { + return "DoubleColumnVector"; + } + return "LongColumnVector"; + } + + private String getOutputWritableType(String primitiveType) throws Exception { + if (primitiveType.equals("long")) { + return "LongWritable"; + } else if (primitiveType.equals("double")) { + return "DoubleWritable"; + } + throw new Exception("Unimplemented primitive output writable: " + primitiveType); + } + + private String getOutputObjectInspector(String primitiveType) throws Exception { + if (primitiveType.equals("long")) { + return "PrimitiveObjectInspectorFactory.writableLongObjectInspector"; + } else if (primitiveType.equals("double")) { + return "PrimitiveObjectInspectorFactory.writableDoubleObjectInspector"; + } + throw new Exception("Unimplemented primitive output inspector: " + primitiveType); + } +} + diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ColumnArithmeticColumn.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ColumnArithmeticColumn.txt new file mode 100644 index 0000000..0e56721 --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ColumnArithmeticColumn.txt @@ -0,0 +1,178 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.*; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +public class extends VectorExpression { + int colNum1; + int colNum2; + int outputColumn; + + public (int colNum1, int colNum2, int outputColumn) { + this.colNum1 = colNum1; + this.colNum2 = colNum2; + this.outputColumn = outputColumn; + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + inputColVector1 = () batch.cols[colNum1]; + inputColVector2 = () batch.cols[colNum2]; + outputColVector = () batch.cols[outputColumn]; + int[] sel = batch.selected; + int n = batch.size; + [] vector1 = inputColVector1.vector; + [] vector2 = inputColVector2.vector; + + [] outputVector = outputColVector.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + /* Set repeating property to false (the default). + * It will be set to true later if needed later. + */ + outputColVector.isRepeating = false; + + //Handle nulls first + if (inputColVector1.noNulls && !inputColVector2.noNulls) { + outputColVector.noNulls = false; + if (inputColVector2.isRepeating) { + //Output will also be repeating and null + outputColVector.isNull[0] = true; + outputColVector.isRepeating = true; + //return as no further processing is needed + return; + } else { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputColVector.isNull[i] = inputColVector2.isNull[i]; + } + } else { + for(int i = 0; i != n; i++) { + outputColVector.isNull[i] = inputColVector2.isNull[i]; + } + } + } + } else if (!inputColVector1.noNulls && inputColVector2.noNulls) { + outputColVector.noNulls = false; + if (inputColVector1.isRepeating) { + //Output will also be repeating and null + outputColVector.isRepeating = true; + outputColVector.isNull[0] = true; + //return as no further processing is needed + return; + } else { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputColVector.isNull[i] = inputColVector1.isNull[i]; + } + } else { + for(int i = 0; i != n; i++) { + outputColVector.isNull[i] = inputColVector1.isNull[i]; + } + } + } + } else if (!inputColVector1.noNulls && !inputColVector2.noNulls) { + outputColVector.noNulls = false; + if (inputColVector1.isRepeating || inputColVector2.isRepeating) { + //Output will also be repeating and null + outputColVector.isRepeating = true; + outputColVector.isNull[0] = true; + //return as no further processing is needed + return; + } else { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputColVector.isNull[i] = inputColVector1.isNull[i] || inputColVector2.isNull[i]; + } + } else { + for(int i = 0; i != n; i++) { + outputColVector.isNull[i] = inputColVector1.isNull[i] || inputColVector2.isNull[i]; + } + } + } + } + + + //Disregard nulls for processing + if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + //All must be selected otherwise size would be zero + //Repeating property will not change. + outputVector[0] = vector1[0] vector2[0]; + outputColVector.isRepeating = true; + } else if (inputColVector1.isRepeating) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = vector1[0] vector2[i]; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = vector1[0] vector2[i]; + } + } + } else if (inputColVector2.isRepeating) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = vector1[i] vector2[0]; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = vector1[i] vector2[0]; + } + } + } else { + if (batch.selectedInUse) { + for(int j=0; j != n; j++) { + int i = sel[j]; + outputVector[i] = vector1[i] vector2[i]; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = vector1[i] vector2[i]; + } + } + } + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return ""; + } +} diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ColumnArithmeticScalar.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ColumnArithmeticScalar.txt new file mode 100644 index 0000000..474a979 --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ColumnArithmeticScalar.txt @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.; +import org.apache.hadoop.hive.ql.exec.vector.; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +public class extends VectorExpression { + private int colNum; + private value; + private int outputColumn; + + public (int colNum, value, int outputColumn) { + this.colNum = colNum; + this.value = value; + this.outputColumn = outputColumn; + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + inputColVector = () batch.cols[colNum]; + outputColVector = () batch.cols[outputColumn]; + int[] sel = batch.selected; + boolean[] inputIsNull = inputColVector.isNull; + boolean[] outputIsNull = outputColVector.isNull; + outputColVector.noNulls = inputColVector.noNulls; + int n = batch.size; + [] vector = inputColVector.vector; + [] outputVector = outputColVector.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector.isRepeating) { + //All must be selected otherwise size would be zero + //Repeating property will not change. + outputVector[0] = vector[0] value; + // Even if there are no nulls, we always copy over entry 0. Simplifies code. + outputIsNull[0] = inputIsNull[0]; + outputColVector.isRepeating = true; + } else if (inputColVector.noNulls) { + if (batch.selectedInUse) { + for(int j=0; j != n; j++) { + int i = sel[j]; + outputVector[i] = vector[i] value; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = vector[i] value; + } + } + outputColVector.isRepeating = false; + } else /* there are nulls */ { + if (batch.selectedInUse) { + for(int j=0; j != n; j++) { + int i = sel[j]; + outputVector[i] = vector[i] value; + outputIsNull[i] = inputIsNull[i]; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = vector[i] value; + } + System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); + } + outputColVector.isRepeating = false; + } + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return ""; + } +} diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ColumnCompareScalar.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ColumnCompareScalar.txt new file mode 100644 index 0000000..dcf87b0 --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ColumnCompareScalar.txt @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.; +import org.apache.hadoop.hive.ql.exec.vector.; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +public class extends VectorExpression { + private int colNum; + private value; + private int outputColumn; + + public (int colNum, value, int outputColumn) { + this.colNum = colNum; + this.value = value; + this.outputColumn = outputColumn; + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + inputColVector = () batch.cols[colNum]; + outputColVector = () batch.cols[outputColumn]; + int[] sel = batch.selected; + boolean[] nullPos = inputColVector.isNull; + int n = batch.size; + [] vector = inputColVector.vector; + [] outputVector = outputColVector.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector.noNulls) { + if (inputColVector.isRepeating) { + //All must be selected otherwise size would be zero + //Repeating property will not change. + outputVector[0] = vector[0] value ? 1 : 0; + outputColVector.isRepeating = true; + } else if (batch.selectedInUse) { + for(int j=0; j != n; j++) { + int i = sel[j]; + outputVector[i] = vector[i] value ? 1 : 0; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = vector[i] value ? 1 : 0; + } + } + } else { + if (inputColVector.isRepeating) { + //All must be selected otherwise size would be zero + //Repeating property will not change. + if (!nullPos[0]) { + outputVector[0] = vector[0] value ? 1 : 0; + } else { + outputVector[0] = 0; + } + outputColVector.isRepeating = true; + } else if (batch.selectedInUse) { + for(int j=0; j != n; j++) { + int i = sel[j]; + if (!nullPos[i]) { + outputVector[i] = vector[i] value ? 1 : 0; + } else { + //compare with null is false + outputVector[i] = 0; + } + } + } else { + for(int i = 0; i != n; i++) { + if (!nullPos[i]) { + outputVector[i] = vector[i] value ? 1 : 0; + } else { + outputVector[i] = 0; + } + } + } + } + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return ""; + } +} diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ColumnUnaryMinus.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ColumnUnaryMinus.txt new file mode 100644 index 0000000..9d16bdd --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ColumnUnaryMinus.txt @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.*; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +public class extends VectorExpression { + int colNum; + int outputColumn; + + public (int colNum, int outputColumn) { + this.colNum = colNum; + this.outputColumn = outputColumn; + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + this.evaluateChildren(batch); + } + + inputColVector = () batch.cols[colNum]; + outputColVector = () batch.cols[outputColumn]; + int[] sel = batch.selected; + boolean[] inputIsNull = inputColVector.isNull; + boolean[] outputIsNull = outputColVector.isNull; + outputColVector.noNulls = inputColVector.noNulls; + int n = batch.size; + [] vector = inputColVector.vector; + [] outputVector = outputColVector.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector.isRepeating) { + //All must be selected otherwise size would be zero + //Repeating property will not change. + outputVector[0] = - vector[0]; + // Even if there are no nulls, we always copy over entry 0. Simplifies code. + outputIsNull[0] = inputIsNull[0]; + outputColVector.isRepeating = true; + } else if (inputColVector.noNulls) { + if (batch.selectedInUse) { + for(int j=0; j != n; j++) { + int i = sel[j]; + outputVector[i] = - vector[i]; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = - vector[i]; + } + } + outputColVector.isRepeating = false; + } else /* there are nulls */ { + if (batch.selectedInUse) { + for(int j=0; j != n; j++) { + int i = sel[j]; + outputVector[i] = - vector[i]; + outputIsNull[i] = inputIsNull[i]; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = - vector[i]; + } + System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); + } + outputColVector.isRepeating = false; + } + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return ""; + } +} diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterColumnCompareColumn.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterColumnCompareColumn.txt new file mode 100644 index 0000000..8e3b717 --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterColumnCompareColumn.txt @@ -0,0 +1,227 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.*; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +public class extends VectorExpression { + int colNum1; + int colNum2; + + public (int colNum1, int colNum2) { + this.colNum1 = colNum1; + this.colNum2 = colNum2; + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + inputColVector1 = () batch.cols[colNum1]; + inputColVector2 = () batch.cols[colNum2]; + int[] sel = batch.selected; + boolean[] nullPos1 = inputColVector1.isNull; + boolean[] nullPos2 = inputColVector2.isNull; + int n = batch.size; + [] vector1 = inputColVector1.vector; + [] vector2 = inputColVector2.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector1.noNulls && inputColVector2.noNulls) { + if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + //All must be selected otherwise size would be zero + //Repeating property will not change. + if (!(vector1[0] vector2[0])) { + batch.size = 0; + } + } else if (inputColVector1.isRepeating) { + if (batch.selectedInUse) { + int newSize = 0; + for(int j=0; j != n; j++) { + int i = sel[j]; + if (vector1[0] vector2[i]) { + sel[newSize++] = i; + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (vector1[0] vector2[i]) { + sel[newSize++] = i; + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else if (inputColVector2.isRepeating) { + if (batch.selectedInUse) { + int newSize = 0; + for(int j=0; j != n; j++) { + int i = sel[j]; + if (vector1[i] vector2[0]) { + sel[newSize++] = i; + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (vector1[i] vector2[0]) { + sel[newSize++] = i; + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else if (batch.selectedInUse) { + int newSize = 0; + for(int j=0; j != n; j++) { + int i = sel[j]; + if (vector1[i] vector2[i]) { + sel[newSize++] = i; + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (vector1[i] vector2[i]) { + sel[newSize++] = i; + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + if (nullPos1[0] || nullPos2[0]) { + batch.size = 0; + } + } else if (inputColVector1.isRepeating) { + if (nullPos1[0]) { + batch.size = 0; + } else { + if (batch.selectedInUse) { + int newSize = 0; + for(int j=0; j != n; j++) { + int i = sel[j]; + if (!nullPos2[i]) { + if (vector1[0] vector2[i]) { + sel[newSize++] = i; + } + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos2[i]) { + if (vector1[0] vector2[i]) { + sel[newSize++] = i; + } + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } + } else if (inputColVector2.isRepeating) { + if (nullPos2[0]) { + batch.size = 0; + } else { + if (batch.selectedInUse) { + int newSize = 0; + for(int j=0; j != n; j++) { + int i = sel[j]; + if (!nullPos1[i]) { + if (vector1[i] vector2[0]) { + sel[newSize++] = i; + } + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos1[i]) { + if (vector1[i] vector2[0]) { + sel[newSize++] = i; + } + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } + } else if (batch.selectedInUse) { + int newSize = 0; + for(int j=0; j != n; j++) { + int i = sel[j]; + if (!nullPos1[i] && !nullPos2[i]) { + if (vector1[i] vector2[i]) { + sel[newSize++] = i; + } + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos1[i] && !nullPos2[i]) { + if (vector1[i] vector2[i]) { + sel[newSize++] = i; + } + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } + + @Override + public String getOutputType() { + return "boolean"; + } + + @Override + public int getOutputColumn() { + return -1; + } +} diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterColumnCompareScalar.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterColumnCompareScalar.txt new file mode 100644 index 0000000..04bc69c --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterColumnCompareScalar.txt @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + + +public class extends VectorExpression { + private int colNum; + private value; + + public (int colNum, value) { + this.colNum = colNum; + this.value = value; + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + inputColVector = () batch.cols[colNum]; + int[] sel = batch.selected; + boolean[] nullPos = inputColVector.isNull; + int n = batch.size; + [] vector = inputColVector.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector.noNulls) { + if (inputColVector.isRepeating) { + //All must be selected otherwise size would be zero + //Repeating property will not change. + if (!(vector[0] value)) { + //Entire batch is filtered out. + batch.size = 0; + } + } else if (batch.selectedInUse) { + int newSize = 0; + for(int j=0; j != n; j++) { + int i = sel[j]; + if (vector[i] value) { + sel[newSize++] = i; + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (vector[i] value) { + sel[newSize++] = i; + } + } + if (newSize < n) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else { + if (inputColVector.isRepeating) { + //All must be selected otherwise size would be zero + //Repeating property will not change. + if (!nullPos[0]) { + if (!(vector[0] value)) { + //Entire batch is filtered out. + batch.size = 0; + } + } else { + batch.size = 0; + } + } else if (batch.selectedInUse) { + int newSize = 0; + for(int j=0; j != n; j++) { + int i = sel[j]; + if (!nullPos[i]) { + if (vector[i] value) { + sel[newSize++] = i; + } + } + } + //Change the selected vector + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos[i]) { + if (vector[i] value) { + sel[newSize++] = i; + } + } + } + if (newSize < n) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } + } + + @Override + public int getOutputColumn() { + return -1; + } + + @Override + public String getOutputType() { + return "boolean"; + } +} diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColumnCompareColumn.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColumnCompareColumn.txt new file mode 100644 index 0000000..9309326 --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColumnCompareColumn.txt @@ -0,0 +1,455 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +/** + * Filter the rows in a batch by comparing one string column to another. + * This code is generated from a template. + */ +public class extends VectorExpression { + private int colNum1; + private int colNum2; + + public (int colNum1, int colNum2) { + this.colNum1 = colNum1; + this.colNum2 = colNum2; + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + BytesColumnVector inputColVector1 = (BytesColumnVector) batch.cols[colNum1]; + BytesColumnVector inputColVector2 = (BytesColumnVector) batch.cols[colNum2]; + int[] sel = batch.selected; + boolean[] nullPos1 = inputColVector1.isNull; + boolean[] nullPos2 = inputColVector2.isNull; + int n = batch.size; + byte[][] vector1 = inputColVector1.vector; + byte[][] vector2 = inputColVector2.vector; + int[] start1 = inputColVector1.start; + int[] start2 = inputColVector2.start; + int[] length1 = inputColVector1.length; + int[] length2 = inputColVector2.length; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + // handle case where neither input has nulls + if (inputColVector1.noNulls && inputColVector2.noNulls) { + if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + + /* Either all must remain selected or all will be eliminated. + * Repeating property will not change. + */ + if (!(StringExpr.compare(vector1[0], start1[0], length1[0], + vector2[0], start2[0], length2[0]) 0)) { + batch.size = 0; + } + } else if (inputColVector1.isRepeating) { + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (StringExpr.compare(vector1[0], start1[0], length1[0], + vector2[i], start2[i], length2[i]) 0) { + sel[newSize++] = i; + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (StringExpr.compare(vector1[0], start1[0], length1[0], + vector2[i], start2[i], length2[i]) 0) { + sel[newSize++] = i; + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else if (inputColVector2.isRepeating) { + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (StringExpr.compare(vector1[i], start1[i], length1[i], + vector2[0], start2[0], length2[0]) 0) { + sel[newSize++] = i; + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (StringExpr.compare(vector1[i], start1[i], length1[i], + vector2[0], start2[0], length2[0]) 0) { + sel[newSize++] = i; + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (StringExpr.compare(vector1[i], start1[i], length1[i], + vector2[i], start2[i], length2[i]) 0) { + sel[newSize++] = i; + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (StringExpr.compare(vector1[i], start1[i], length1[i], + vector2[i], start2[i], length2[i]) 0) { + sel[newSize++] = i; + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + + // handle case where only input 2 has nulls + } else if (inputColVector1.noNulls) { + if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + if (nullPos2[0] || + !(StringExpr.compare(vector1[0], start1[0], length1[0], + vector2[0], start2[0], length2[0]) 0)) { + batch.size = 0; + } + } else if (inputColVector1.isRepeating) { + + // no need to check for nulls in input 1 + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos2[i]) { + if (StringExpr.compare(vector1[0], start1[0], length1[0], + vector2[i], start2[i], length2[i]) 0) { + sel[newSize++] = i; + } + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos2[i]) { + if (StringExpr.compare(vector1[0], start1[0], length1[0], + vector2[i], start2[i], length2[i]) 0) { + sel[newSize++] = i; + } + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else if (inputColVector2.isRepeating) { + if (nullPos2[0]) { + + // no values will qualify because every comparison will be with NULL + batch.size = 0; + return; + } + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (StringExpr.compare(vector1[i], start1[i], length1[i], + vector2[0], start2[0], length2[0]) 0) { + sel[newSize++] = i; + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (StringExpr.compare(vector1[i], start1[i], length1[i], + vector2[0], start2[0], length2[0]) 0) { + sel[newSize++] = i; + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else { // neither input is repeating + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos2[i]) { + if (StringExpr.compare(vector1[i], start1[i], length1[i], + vector2[i], start2[i], length2[i]) 0) { + sel[newSize++] = i; + } + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos2[i]) { + if (StringExpr.compare(vector1[i], start1[i], length1[i], + vector2[i], start2[i], length2[i]) 0) { + sel[newSize++] = i; + } + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } + + // handle case where only input 1 has nulls + } else if (inputColVector2.noNulls) { + if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + if (nullPos1[0] || + !(StringExpr.compare(vector1[0], start1[0], length1[0], + vector2[0], start2[0], length2[0]) 0)) { + batch.size = 0; + return; + } + } else if (inputColVector1.isRepeating) { + if (nullPos1[0]) { + + // if repeating value is null then every comparison will fail so nothing qualifies + batch.size = 0; + return; + } + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (StringExpr.compare(vector1[0], start1[0], length1[0], + vector2[i], start2[i], length2[i]) 0) { + sel[newSize++] = i; + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (StringExpr.compare(vector1[0], start1[0], length1[0], + vector2[i], start2[i], length2[i]) 0) { + sel[newSize++] = i; + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else if (inputColVector2.isRepeating) { + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos1[i]) { + if (StringExpr.compare(vector1[i], start1[i], length1[i], + vector2[0], start2[0], length2[0]) 0) { + sel[newSize++] = i; + } + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos2[i]) { + if (StringExpr.compare(vector1[i], start1[i], length1[i], + vector2[0], start2[0], length2[0]) 0) { + sel[newSize++] = i; + } + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else { // neither input is repeating + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos1[i]) { + if (StringExpr.compare(vector1[i], start1[i], length1[i], + vector2[i], start2[i], length2[i]) 0) { + sel[newSize++] = i; + } + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos1[i]) { + if (StringExpr.compare(vector1[i], start1[i], length1[i], + vector2[i], start2[i], length2[i]) 0) { + sel[newSize++] = i; + } + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } + + // handle case where both inputs have nulls + } else { + if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + if (nullPos1[0] || nullPos2[0] || + !(StringExpr.compare(vector1[0], start1[0], length1[0], + vector2[0], start2[0], length2[0]) 0)) { + batch.size = 0; + } + } else if (inputColVector1.isRepeating) { + if (nullPos1[0]) { + batch.size = 0; + return; + } + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos2[i]) { + if (StringExpr.compare(vector1[0], start1[0], length1[0], + vector2[i], start2[i], length2[i]) 0) { + sel[newSize++] = i; + } + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos2[i]) { + if (StringExpr.compare(vector1[0], start1[0], length1[0], + vector2[i], start2[i], length2[i]) 0) { + sel[newSize++] = i; + } + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else if (inputColVector2.isRepeating) { + if (nullPos2[0]) { + batch.size = 0; + return; + } + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos1[i]) { + if (StringExpr.compare(vector1[i], start1[i], length1[i], + vector2[0], start2[0], length2[0]) 0) { + sel[newSize++] = i; + } + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos1[i]) { + if (StringExpr.compare(vector1[i], start1[i], length1[i], + vector2[0], start2[0], length2[0]) 0) { + sel[newSize++] = i; + } + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else { // neither input is repeating + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos1[i] && !nullPos2[i]) { + if (StringExpr.compare(vector1[i], start1[i], length1[i], + vector2[i], start2[i], length2[i]) 0) { + sel[newSize++] = i; + } + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos1[i] && !nullPos2[i]) { + if (StringExpr.compare(vector1[i], start1[i], length1[i], + vector2[i], start2[i], length2[i]) 0) { + sel[newSize++] = i; + } + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } + } + } + + @Override + public String getOutputType() { + return "boolean"; + } + + @Override + public int getOutputColumn() { + return -1; + } +} diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColumnCompareScalar.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColumnCompareScalar.txt new file mode 100644 index 0000000..30eddcf --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColumnCompareScalar.txt @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +/** + * This is a generated class to evaluate a comparison on a vector of strings. + */ +public class extends VectorExpression { + private int colNum; + private byte[] value; + + public (int colNum, byte[] value) { + this.colNum = colNum; + this.value = value; + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + if (childExpressions != null) { + super.evaluateChildren(batch); + } + BytesColumnVector inputColVector = (BytesColumnVector) batch.cols[colNum]; + int[] sel = batch.selected; + boolean[] nullPos = inputColVector.isNull; + int n = batch.size; + byte[][] vector = inputColVector.vector; + int[] length = inputColVector.length; + int[] start = inputColVector.start; + + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector.noNulls) { + if (inputColVector.isRepeating) { + + // All must be selected otherwise size would be zero. Repeating property will not change. + if (!(StringExpr.compare(vector[0], start[0], length[0], value, 0, value.length) 0)) { + + //Entire batch is filtered out. + batch.size = 0; + } + } else if (batch.selectedInUse) { + int newSize = 0; + for(int j=0; j != n; j++) { + int i = sel[j]; + if (StringExpr.compare(vector[i], start[i], length[i], value, 0, value.length) 0) { + sel[newSize++] = i; + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (StringExpr.compare(vector[i], start[i], length[i], value, 0, value.length) 0) { + sel[newSize++] = i; + } + } + if (newSize < n) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else { + if (inputColVector.isRepeating) { + + // All must be selected otherwise size would be zero. Repeating property will not change. + if (!nullPos[0]) { + if (!(StringExpr.compare(vector[0], start[0], length[0], value, 0, value.length) 0)) { + + //Entire batch is filtered out. + batch.size = 0; + } + } else { + batch.size = 0; + } + } else if (batch.selectedInUse) { + int newSize = 0; + for(int j=0; j != n; j++) { + int i = sel[j]; + if (!nullPos[i]) { + if (StringExpr.compare(vector[i], start[i], length[i], value, 0, value.length) 0) { + sel[newSize++] = i; + } + } + } + + //Change the selected vector + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos[i]) { + if (StringExpr.compare(vector[i], start[i], length[i], value, 0, value.length) 0) { + sel[newSize++] = i; + } + } + } + if (newSize < n) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } + } + + @Override + public int getOutputColumn() { + return -1; + } + + @Override + public String getOutputType() { + return "boolean"; + } +} diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ScalarArithmeticColumn.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ScalarArithmeticColumn.txt new file mode 100644 index 0000000..9899e7e --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/ScalarArithmeticColumn.txt @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; + +/* + * Because of the templatized nature of the code, either or both + * of these ColumnVector imports may be needed. Listing both of them + * rather than using ....vectorization.*; + */ +import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +/** + * Implements a vectorized arithmetic operator with a scalar on the left and a + * column vector on the right. The result is output to an output column vector. + */ +public class extends VectorExpression { + private int colNum; + private value; + private int outputColumn; + + public ( value, int colNum, int outputColumn) { + this.colNum = colNum; + this.value = value; + this.outputColumn = outputColumn; + } + + @Override + /** + * Method to evaluate scalar-column operation in vectorized fashion. + * + * @batch a package of rows with each column stored in a vector + */ + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + inputColVector = () batch.cols[colNum]; + outputColVector = () batch.cols[outputColumn]; + int[] sel = batch.selected; + boolean[] inputIsNull = inputColVector.isNull; + boolean[] outputIsNull = outputColVector.isNull; + outputColVector.noNulls = inputColVector.noNulls; + int n = batch.size; + [] vector = inputColVector.vector; + [] outputVector = outputColVector.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector.isRepeating) { + + /* + * All must be selected otherwise size would be zero + * Repeating property will not change. + */ + outputVector[0] = value vector[0]; + + // Even if there are no nulls, we always copy over entry 0. Simplifies code. + outputIsNull[0] = inputIsNull[0]; + outputColVector.isRepeating = true; + } else if (inputColVector.noNulls) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = value vector[i]; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = value vector[i]; + } + } + outputColVector.isRepeating = false; + } else { /* there are nulls */ + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = value vector[i]; + outputIsNull[i] = inputIsNull[i]; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = value vector[i]; + } + System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); + } + outputColVector.isRepeating = false; + } + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return ""; + } +} diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestClass.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestClass.txt new file mode 100644 index 0000000..c8de5de --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestClass.txt @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import static org.junit.Assert.assertEquals; +import java.util.Random; +import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.util.VectorizedRowGroupGenUtil; +import org.junit.Test; + + +/** + * + * . + * + */ +public class { + + private static final int BATCH_SIZE = 100; + private static final long SEED = 0xfa57; + + + +} + + diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestCodeGen.java ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestCodeGen.java new file mode 100644 index 0000000..94bbe36 --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestCodeGen.java @@ -0,0 +1,256 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import java.io.IOException; +import java.util.HashMap; + +/** + * + * TestCodeGen. + * This class is mutable and maintains a hashmap of TestSuiteClassName to test cases. + * The tests cases are added over the course of vectorized expressions class generation, + * with test classes being outputted at the end. For each column vector (inputs and/or outputs) + * a matrix of pairwise covering Booleans is used to generate test cases across nulls and + * repeating dimensions. Based on the input column vector(s) nulls and repeating states + * the states of the output column vector (if there is one) is validated, along with the null + * vector. For filter operations the selection vector is validated against the generated + * data. Each template corresponds to a class representing a test suite. + */ +public class TestCodeGen { + + public enum TestSuiteClassName{ + TestColumnScalarOperationVectorExpressionEvaluation, + TestColumnScalarFilterVectorExpressionEvaluation, + TestColumnColumnOperationVectorExpressionEvaluation, + TestColumnColumnFilterVectorExpressionEvaluation, + } + + private final String testOutputDir; + private final String templateDirectory; + private final HashMap testsuites; + + public TestCodeGen(String testOutputDir, String templateDirectory) { + this.testOutputDir = testOutputDir; + this.templateDirectory = templateDirectory; + testsuites = new HashMap(); + + for(TestSuiteClassName className : TestSuiteClassName.values()) { + testsuites.put(className,new StringBuilder()); + } + + } + + public void addColumnScalarOperationTestCases(boolean op1IsCol, String vectorExpClassName, + String inputColumnVectorType, String outputColumnVectorType, String scalarType) + throws IOException { + + TestSuiteClassName template = + TestSuiteClassName.TestColumnScalarOperationVectorExpressionEvaluation; + + //Read the template into a string; + String templateFile = CodeGen.joinPath(this.templateDirectory,template.toString()+".txt"); + String templateString = removeTemplateComments(CodeGen.readFile(templateFile)); + + for(Boolean[] testMatrix :new Boolean[][]{ + // Pairwise: InitOuputColHasNulls, InitOuputColIsRepeating, ColumnHasNulls, ColumnIsRepeating + {false, true, true, true}, + {false, false, false, false}, + {true, false, true, false}, + {true, true, false, false}, + {true, false, false, true}}) { + String testCase = templateString; + testCase = testCase.replaceAll("", + "test" + + vectorExpClassName + + createNullRepeatingNameFragment("Out", testMatrix[0], testMatrix[1]) + + createNullRepeatingNameFragment("Col", testMatrix[2], testMatrix[3])); + testCase = testCase.replaceAll("", vectorExpClassName); + testCase = testCase.replaceAll("", inputColumnVectorType); + testCase = testCase.replaceAll("", outputColumnVectorType); + testCase = testCase.replaceAll("", scalarType); + testCase = testCase.replaceAll("", CodeGen.getCamelCaseType(scalarType)); + testCase = testCase.replaceAll("", testMatrix[0].toString()); + testCase = testCase.replaceAll("", testMatrix[1].toString()); + testCase = testCase.replaceAll("", testMatrix[2].toString()); + testCase = testCase.replaceAll("", testMatrix[3].toString()); + + if(op1IsCol){ + testCase = testCase.replaceAll("","0, scalarValue"); + }else{ + testCase = testCase.replaceAll("","scalarValue, 0"); + } + + testsuites.get(template).append(testCase); + } + } + + public void addColumnScalarFilterTestCases(boolean op1IsCol, String vectorExpClassName, + String inputColumnVectorType, String scalarType, String operatorSymbol) + throws IOException { + + TestSuiteClassName template = + TestSuiteClassName.TestColumnScalarFilterVectorExpressionEvaluation; + + //Read the template into a string; + String templateFile = CodeGen.joinPath(this.templateDirectory,template.toString()+".txt"); + String templateString = removeTemplateComments(CodeGen.readFile(templateFile)); + + for(Boolean[] testMatrix : new Boolean[][]{ + // Pairwise: ColumnHasNulls, ColumnIsRepeating + {true, true}, + {true, false}, + {false, false}, + {false, true}}) { + String testCase = templateString; + testCase = testCase.replaceAll("", + "test" + + vectorExpClassName + + createNullRepeatingNameFragment("Col", testMatrix[0], testMatrix[1])); + testCase = testCase.replaceAll("", vectorExpClassName); + testCase = testCase.replaceAll("", inputColumnVectorType); + testCase = testCase.replaceAll("", scalarType); + testCase = testCase.replaceAll("", CodeGen.getCamelCaseType(scalarType)); + testCase = testCase.replaceAll("", testMatrix[0].toString()); + testCase = testCase.replaceAll("", testMatrix[1].toString()); + testCase = testCase.replaceAll("", operatorSymbol); + + if(op1IsCol){ + testCase = testCase.replaceAll("","0, scalarValue"); + }else{ + testCase = testCase.replaceAll("","scalarValue, 0"); + } + + testsuites.get(template).append(testCase); + } + } + + public void addColumnColumnOperationTestCases(String vectorExpClassName, + String inputColumnVectorType1, String inputColumnVectorType2, String outputColumnVectorType) + throws IOException { + + TestSuiteClassName template= + TestSuiteClassName.TestColumnColumnOperationVectorExpressionEvaluation; + + //Read the template into a string; + String templateFile = CodeGen.joinPath(this.templateDirectory,template.toString()+".txt"); + String templateString = removeTemplateComments(CodeGen.readFile(templateFile)); + + for(Boolean[] testMatrix : new Boolean[][]{ + // Pairwise: InitOuputColHasNulls, InitOuputColIsRepeating, Column1HasNulls, + // Column1IsRepeating, Column2HasNulls, Column2IsRepeating + {true, true, false, true, true, true}, + {false, false, true, false, false, false}, + {true, false, true, false, true, true}, + {true, true, true, true, false, false}, + {false, false, false, true, true, false}, + {false, true, false, false, false, true}}) { + String testCase = templateString; + testCase = testCase.replaceAll("", + "test" + + vectorExpClassName + + createNullRepeatingNameFragment("Out", testMatrix[0], testMatrix[1]) + + createNullRepeatingNameFragment("C1", testMatrix[2], testMatrix[3]) + + createNullRepeatingNameFragment("C2", testMatrix[4], testMatrix[5])); + testCase = testCase.replaceAll("", vectorExpClassName); + testCase = testCase.replaceAll("", inputColumnVectorType1); + testCase = testCase.replaceAll("", inputColumnVectorType2); + testCase = testCase.replaceAll("", outputColumnVectorType); + testCase = testCase.replaceAll("", testMatrix[0].toString()); + testCase = testCase.replaceAll("", testMatrix[1].toString()); + testCase = testCase.replaceAll("", testMatrix[2].toString()); + testCase = testCase.replaceAll("", testMatrix[3].toString()); + testCase = testCase.replaceAll("", testMatrix[4].toString()); + testCase = testCase.replaceAll("", testMatrix[5].toString()); + + testsuites.get(template).append(testCase); + } + } + + public void addColumnColumnFilterTestCases(String vectorExpClassName, + String inputColumnVectorType1, String inputColumnVectorType2, String operatorSymbol) + throws IOException { + + TestSuiteClassName template= + TestSuiteClassName.TestColumnColumnFilterVectorExpressionEvaluation; + + //Read the template into a string; + String templateFile = CodeGen.joinPath(this.templateDirectory,template.toString()+".txt"); + String templateString = removeTemplateComments(CodeGen.readFile(templateFile)); + + for(Boolean[] testMatrix : new Boolean[][]{ + // Pairwise: Column1HasNulls, Column1IsRepeating, Column2HasNulls, Column2IsRepeating + {false, true, true, true}, + {false, false, false, false}, + {true, false, true, false}, + {true, true, false, false}, + {true, false, false, true}}) { + String testCase = templateString; + testCase = testCase.replaceAll("", + "test" + + vectorExpClassName + + createNullRepeatingNameFragment("C1", testMatrix[0], testMatrix[1]) + + createNullRepeatingNameFragment("C2", testMatrix[2], testMatrix[3])); + testCase = testCase.replaceAll("", vectorExpClassName); + testCase = testCase.replaceAll("", inputColumnVectorType1); + testCase = testCase.replaceAll("", inputColumnVectorType2); + testCase = testCase.replaceAll("", testMatrix[0].toString()); + testCase = testCase.replaceAll("", testMatrix[1].toString()); + testCase = testCase.replaceAll("", testMatrix[2].toString()); + testCase = testCase.replaceAll("", testMatrix[3].toString()); + testCase = testCase.replaceAll("", operatorSymbol); + + testsuites.get(template).append(testCase); + } + } + + public void generateTestSuites() throws IOException { + + String templateFile = CodeGen.joinPath(this.templateDirectory, "TestClass.txt"); + for(TestSuiteClassName testClass : testsuites.keySet()) { + + String templateString = CodeGen.readFile(templateFile); + templateString = templateString.replaceAll("", testClass.toString()); + templateString = templateString.replaceAll("", testsuites.get(testClass).toString()); + + String outputFile = CodeGen.joinPath(this.testOutputDir, testClass + ".java"); + + CodeGen.writeFile(outputFile, templateString); + } + } + + private static String createNullRepeatingNameFragment(String idenitfier, boolean nulls, boolean repeating) + { + if(nulls || repeating){ + if(nulls){ + idenitfier+="Nulls"; + } + if(repeating){ + idenitfier+="Repeats"; + } + return idenitfier; + } + + return ""; + } + + private static String removeTemplateComments(String templateString){ + return templateString.replaceAll("(?s)", ""); + } +} diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnColumnFilterVectorExpressionEvaluation.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnColumnFilterVectorExpressionEvaluation.txt new file mode 100644 index 0000000..2bb1aa3 --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnColumnFilterVectorExpressionEvaluation.txt @@ -0,0 +1,69 @@ + + @Test + public void () { + + Random rand = new Random(SEED); + + inputColumnVector1 = + VectorizedRowGroupGenUtil.generate(, + , BATCH_SIZE, rand); + + inputColumnVector2 = + VectorizedRowGroupGenUtil.generate(, + , BATCH_SIZE, rand); + + VectorizedRowBatch rowBatch = new VectorizedRowBatch(2, BATCH_SIZE); + rowBatch.cols[0] = inputColumnVector1; + rowBatch.cols[1] = inputColumnVector2; + + vectorExpression = + new (0, 1); + + vectorExpression.evaluate(rowBatch); + + int selectedIndex = 0; + for(int i = 0; i < BATCH_SIZE; i++) { + //null vector is safe to check, as it is always initialized to match the data vector + if(!inputColumnVector1.isNull[i] && !inputColumnVector2.isNull[i]) { + if(inputColumnVector1.vector[i] inputColumnVector2.vector[i]) { + assertEquals( + "Vector index that passes filter " + + inputColumnVector1.vector[i] + "" + + inputColumnVector2.vector[i] + " is not in rowBatch selected index", + i, + rowBatch.selected[selectedIndex]); + selectedIndex++; + } + } + } + + assertEquals("Row batch size not set to number of selected rows: " + selectedIndex, + selectedIndex, rowBatch.size); + + if(selectedIndex > 0 && selectedIndex < BATCH_SIZE) { + assertEquals( + "selectedInUse should be set when > 0 and < entire batch(" + BATCH_SIZE + ") is selected: " + + selectedIndex, + true, rowBatch.selectedInUse); + } else if(selectedIndex == BATCH_SIZE) { + assertEquals( + "selectedInUse should not be set when entire batch(" + BATCH_SIZE + ") is selected: " + + selectedIndex, + false, rowBatch.selectedInUse); + } + } \ No newline at end of file diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnColumnOperationVectorExpressionEvaluation.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnColumnOperationVectorExpressionEvaluation.txt new file mode 100644 index 0000000..4ab3e76 --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnColumnOperationVectorExpressionEvaluation.txt @@ -0,0 +1,64 @@ + + @Test + public void () { + + Random rand = new Random(SEED); + + outputColumnVector = + VectorizedRowGroupGenUtil.generate(, + , BATCH_SIZE, rand); + + inputColumnVector1 = + VectorizedRowGroupGenUtil.generate(, + , BATCH_SIZE, rand); + + inputColumnVector2 = + VectorizedRowGroupGenUtil.generate(, + , BATCH_SIZE, rand); + + VectorizedRowBatch rowBatch = new VectorizedRowBatch(3, BATCH_SIZE); + rowBatch.cols[0] = inputColumnVector1; + rowBatch.cols[1] = inputColumnVector2; + rowBatch.cols[2] = outputColumnVector; + + vectorExpression = + new (0, 1, 2); + + vectorExpression.evaluate(rowBatch); + + assertEquals( + "Output column vector repeating state does not match operand columns", + (!inputColumnVector1.noNulls && inputColumnVector1.isRepeating) + || (!inputColumnVector2.noNulls && inputColumnVector2.isRepeating) + || inputColumnVector1.isRepeating && inputColumnVector2.isRepeating, + outputColumnVector.isRepeating); + + assertEquals( + "Output column vector no nulls state does not match operand columns", + inputColumnVector1.noNulls && inputColumnVector2.noNulls, outputColumnVector.noNulls); + + //if repeating, only the first value matters + if(!outputColumnVector.noNulls && !outputColumnVector.isRepeating) { + for(int i = 0; i < BATCH_SIZE; i++) { + //null vectors are safe to check, as they are always initialized to match the data vector + assertEquals("Output vector doesn't match input vectors' is null state for index", + inputColumnVector1.isNull[i] || inputColumnVector2.isNull[i], + outputColumnVector.isNull[i]); + } + } + } \ No newline at end of file diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnScalarFilterVectorExpressionEvaluation.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnScalarFilterVectorExpressionEvaluation.txt new file mode 100644 index 0000000..5b53d6a --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnScalarFilterVectorExpressionEvaluation.txt @@ -0,0 +1,76 @@ + + @Test + public void () { + + Random rand = new Random(SEED); + + inputColumnVector = + VectorizedRowGroupGenUtil.generate(, + , BATCH_SIZE, rand); + + VectorizedRowBatch rowBatch = new VectorizedRowBatch(1, BATCH_SIZE); + rowBatch.cols[0] = inputColumnVector; + + scalarValue = 0; + do { + scalarValue = rand.next(); + } while(scalarValue == 0); + + vectorExpression = + new (); + + vectorExpression.evaluate(rowBatch); + + int selectedIndex = 0; + //check for isRepeating optimization + if(inputColumnVector.isRepeating) { + //null vector is safe to check, as it is always initialized to match the data vector + selectedIndex = + !inputColumnVector.isNull[0] && inputColumnVector.vector[0] scalarValue + ? BATCH_SIZE : 0; + } else { + for(int i = 0; i < BATCH_SIZE; i++) { + if(!inputColumnVector.isNull[i]) { + if(inputColumnVector.vector[i] scalarValue) { + assertEquals( + "Vector index that passes filter " + + inputColumnVector.vector[i] + "" + + scalarValue + " is not in rowBatch selected index", + i, + rowBatch.selected[selectedIndex]); + selectedIndex++; + } + } + } + } + + assertEquals("Row batch size not set to number of selected rows: " + selectedIndex, + selectedIndex, rowBatch.size); + + if(selectedIndex > 0 && selectedIndex < BATCH_SIZE) { + assertEquals( + "selectedInUse should be set when > 0 and < entire batch(" + BATCH_SIZE + ") is selected: " + + selectedIndex, + true, rowBatch.selectedInUse); + } else if(selectedIndex == BATCH_SIZE) { + assertEquals( + "selectedInUse should not be set when entire batch(" + BATCH_SIZE + ") is selected: " + + selectedIndex, + false, rowBatch.selectedInUse); + } + } \ No newline at end of file diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnScalarOperationVectorExpressionEvaluation.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnScalarOperationVectorExpressionEvaluation.txt new file mode 100644 index 0000000..e5f3f18 --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnScalarOperationVectorExpressionEvaluation.txt @@ -0,0 +1,59 @@ + + @Test + public void () { + + Random rand = new Random(SEED); + + outputColumnVector = + VectorizedRowGroupGenUtil.generate(, + , BATCH_SIZE, rand); + + inputColumnVector = + VectorizedRowGroupGenUtil.generate(, + , BATCH_SIZE, rand); + + VectorizedRowBatch rowBatch = new VectorizedRowBatch(2, BATCH_SIZE); + rowBatch.cols[0] = inputColumnVector; + rowBatch.cols[1] = outputColumnVector; + + scalarValue = 0; + do { + scalarValue = rand.next(); + } while(scalarValue == 0); + + vectorExpression = + new (, 1); + + vectorExpression.evaluate(rowBatch); + + assertEquals( + "Output column vector is repeating state does not match operand column", + inputColumnVector.isRepeating, outputColumnVector.isRepeating); + + assertEquals( + "Output column vector no nulls state does not match operand column", + inputColumnVector.noNulls, outputColumnVector.noNulls); + + if(!outputColumnVector.noNulls && !outputColumnVector.isRepeating) { + for(int i = 0; i < BATCH_SIZE; i++) { + //null vectors are safe to check, as they are always initialized to match the data vector + assertEquals("Output vector doesn't match input vector's is null state for index", + inputColumnVector.isNull[i], outputColumnVector.isNull[i]); + } + } + } \ No newline at end of file diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFAvg.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFAvg.txt new file mode 100644 index 0000000..d85346d --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFAvg.txt @@ -0,0 +1,440 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen; + +import java.util.ArrayList; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates. + VectorAggregateExpression.AggregationBuffer; +import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; + +import org.apache.hadoop.hive.ql.io.orc.*; + +@Description(name = "avg", value = "_FUNC_(expr) - Returns the average value of expr (vectorized, type: )") +public class extends VectorAggregateExpression { + + /** class for storing the current aggregate value. */ + static class Aggregation implements AggregationBuffer { + sum; + long count; + boolean isNull; + + public void sumValue( value) { + if (isNull) { + sum = value; + count = 1; + isNull = false; + } else { + sum += value; + count++; + } + } + } + + private VectorExpression inputExpression; + private Object[] partialResult; + private LongWritable resultCount; + private DoubleWritable resultSum; + private StructObjectInspector soi; + + public (VectorExpression inputExpression) { + super(); + this.inputExpression = inputExpression; + partialResult = new Object[2]; + resultCount = new LongWritable(); + resultSum = new DoubleWritable(); + partialResult[0] = resultCount; + partialResult[1] = resultSum; + + initPartialResultInspector(); + } + + private void initPartialResultInspector() { + ArrayList foi = new ArrayList(); + foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector); + foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); + ArrayList fname = new ArrayList(); + fname.add("count"); + fname.add("sum"); + soi = ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi); + } + + private Aggregation getCurrentAggregationBuffer( + VectorAggregationBufferRow[] aggregationBufferSets, + int bufferIndex, + int row) { + VectorAggregationBufferRow mySet = aggregationBufferSets[row]; + Aggregation myagg = (Aggregation) mySet.getAggregationBuffer(bufferIndex); + return myagg; + } + + @Override + public void aggregateInputSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int bufferIndex, + VectorizedRowBatch batch) throws HiveException { + + int batchSize = batch.size; + + if (batchSize == 0) { + return; + } + + inputExpression.evaluate(batch); + + LongColumnVector inputVector = (LongColumnVector)batch. + cols[this.inputExpression.getOutputColumn()]; + long[] vector = inputVector.vector; + + if (inputVector.noNulls) { + if (inputVector.isRepeating) { + iterateNoNullsRepeatingWithAggregationSelection( + aggregationBufferSets, bufferIndex, + vector[0], batchSize); + } else { + if (batch.selectedInUse) { + iterateNoNullsSelectionWithAggregationSelection( + aggregationBufferSets, bufferIndex, + vector, batch.selected, batchSize); + } else { + iterateNoNullsWithAggregationSelection( + aggregationBufferSets, bufferIndex, + vector, batchSize); + } + } + } else { + if (inputVector.isRepeating) { + if (batch.selectedInUse) { + iterateHasNullsRepeatingSelectionWithAggregationSelection( + aggregationBufferSets, bufferIndex, + vector[0], batchSize, batch.selected, inputVector.isNull); + } else { + iterateHasNullsRepeatingWithAggregationSelection( + aggregationBufferSets, bufferIndex, + vector[0], batchSize, inputVector.isNull); + } + } else { + if (batch.selectedInUse) { + iterateHasNullsSelectionWithAggregationSelection( + aggregationBufferSets, bufferIndex, + vector, batchSize, batch.selected, inputVector.isNull); + } else { + iterateHasNullsWithAggregationSelection( + aggregationBufferSets, bufferIndex, + vector, batchSize, inputVector.isNull); + } + } + } + } + + private void iterateNoNullsRepeatingWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int bufferIndex, + long value, + int batchSize) { + + for (int i=0; i < batchSize; ++i) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + bufferIndex, + i); + myagg.sumValue(value); + } + } + + private void iterateNoNullsSelectionWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int bufferIndex, + long[] values, + int[] selection, + int batchSize) { + + for (int i=0; i < batchSize; ++i) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + bufferIndex, + i); + myagg.sumValue(values[selection[i]]); + } + } + + private void iterateNoNullsWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int bufferIndex, + long[] values, + int batchSize) { + for (int i=0; i < batchSize; ++i) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + bufferIndex, + i); + myagg.sumValue(values[i]); + } + } + + private void iterateHasNullsRepeatingSelectionWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int bufferIndex, + long value, + int batchSize, + int[] selection, + boolean[] isNull) { + + for (int i=0; i < batchSize; ++i) { + if (!isNull[selection[i]]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + bufferIndex, + i); + myagg.sumValue(value); + } + } + + } + + private void iterateHasNullsRepeatingWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int bufferIndex, + long value, + int batchSize, + boolean[] isNull) { + + for (int i=0; i < batchSize; ++i) { + if (!isNull[i]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + bufferIndex, + i); + myagg.sumValue(value); + } + } + } + + private void iterateHasNullsSelectionWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int bufferIndex, + long[] values, + int batchSize, + int[] selection, + boolean[] isNull) { + + for (int j=0; j < batchSize; ++j) { + int i = selection[j]; + if (!isNull[i]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + bufferIndex, + j); + myagg.sumValue(values[i]); + } + } + } + + private void iterateHasNullsWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int bufferIndex, + long[] values, + int batchSize, + boolean[] isNull) { + + for (int i=0; i < batchSize; ++i) { + if (!isNull[i]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + bufferIndex, + i); + myagg.sumValue(values[i]); + } + } + } + + + @Override + public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) throws HiveException { + + inputExpression.evaluate(batch); + + inputVector = ()batch.cols[this.inputExpression.getOutputColumn()]; + + int batchSize = batch.size; + + if (batchSize == 0) { + return; + } + + Aggregation myagg = (Aggregation)agg; + + [] vector = inputVector.vector; + + if (inputVector.isRepeating) { + if (inputVector.noNulls || !inputVector.isNull[0]) { + if (myagg.isNull) { + myagg.isNull = false; + myagg.sum = 0; + myagg.count = 0; + } + myagg.sum += vector[0]*batchSize; + myagg.count += batchSize; + } + return; + } + + if (!batch.selectedInUse && inputVector.noNulls) { + iterateNoSelectionNoNulls(myagg, vector, batchSize); + } + else if (!batch.selectedInUse) { + iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull); + } + else if (inputVector.noNulls){ + iterateSelectionNoNulls(myagg, vector, batchSize, batch.selected); + } + else { + iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, batch.selected); + } + } + + private void iterateSelectionHasNulls( + Aggregation myagg, + [] vector, + int batchSize, + boolean[] isNull, + int[] selected) { + + for (int j=0; j< batchSize; ++j) { + int i = selected[j]; + if (!isNull[i]) { + value = vector[i]; + if (myagg.isNull) { + myagg.isNull = false; + myagg.sum = 0; + myagg.count = 0; + } + myagg.sum += value; + myagg.count += 1; + } + } + } + + private void iterateSelectionNoNulls( + Aggregation myagg, + [] vector, + int batchSize, + int[] selected) { + + if (myagg.isNull) { + myagg.isNull = false; + myagg.sum = 0; + myagg.count = 0; + } + + for (int i=0; i< batchSize; ++i) { + value = vector[selected[i]]; + myagg.sum += value; + myagg.count += 1; + } + } + + private void iterateNoSelectionHasNulls( + Aggregation myagg, + [] vector, + int batchSize, + boolean[] isNull) { + + for(int i=0;i value = vector[i]; + if (myagg.isNull) { + myagg.isNull = false; + myagg.sum = 0; + myagg.count = 0; + } + myagg.sum += value; + myagg.count += 1; + } + } + } + + private void iterateNoSelectionNoNulls( + Aggregation myagg, + [] vector, + int batchSize) { + if (myagg.isNull) { + myagg.isNull = false; + myagg.sum = 0; + myagg.count = 0; + } + + for (int i=0;i value = vector[i]; + myagg.sum += value; + myagg.count += 1; + } + } + + @Override + public AggregationBuffer getNewAggregationBuffer() throws HiveException { + return new Aggregation(); + } + + @Override + public void reset(AggregationBuffer agg) throws HiveException { + Aggregation myAgg = (Aggregation) agg; + myAgg.isNull = true; + } + + @Override + public Object evaluateOutput( + AggregationBuffer agg) throws HiveException { + Aggregation myagg = (Aggregation) agg; + if (myagg.isNull) { + return null; + } + else { + assert(0 < myagg.count); + resultCount.set (myagg.count); + resultSum.set (myagg.sum); + return partialResult; + } + } + + @Override + public ObjectInspector getOutputObjectInspector() { + return soi; + } +} + diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFCount.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFCount.txt new file mode 100644 index 0000000..54c80ec --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFCount.txt @@ -0,0 +1,262 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen; + +import java.util.ArrayList; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates. + VectorAggregateExpression.AggregationBuffer; +import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + +/** +* . Vectorized implementation for COUNT aggregates. +*/ +@Description(name = "count", value = "_FUNC_(expr) - Returns the maximum value of expr (vectorized, type: )") +public class extends VectorAggregateExpression { + + /** + /* class for storing the current aggregate value. + */ + static class Aggregation implements AggregationBuffer { + long value; + boolean isNull; + + public void initIfNull() { + if (isNull) { + isNull = false; + value = 0; + } + } + } + + private VectorExpression inputExpression; + private LongWritable result; + + public (VectorExpression inputExpression) { + super(); + this.inputExpression = inputExpression; + result = new LongWritable(0); + } + + private Aggregation getCurrentAggregationBuffer( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + int row) { + VectorAggregationBufferRow mySet = aggregationBufferSets[row]; + Aggregation myagg = (Aggregation) mySet.getAggregationBuffer(aggregateIndex); + return myagg; + } + + @Override + public void aggregateInputSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + VectorizedRowBatch batch) throws HiveException { + + int batchSize = batch.size; + + if (batchSize == 0) { + return; + } + + inputExpression.evaluate(batch); + + inputVector = ()batch. + cols[this.inputExpression.getOutputColumn()]; + + if (inputVector.noNulls) { + // if there are no nulls then the iteration is the same on all cases + iterateNoNullsWithAggregationSelection( + aggregationBufferSets, aggregateIndex, batchSize); + } else if (!batch.selectedInUse) { + iterateHasNullsWithAggregationSelection( + aggregationBufferSets, aggregateIndex, + batchSize, inputVector.isNull); + } else if (batch.selectedInUse) { + iterateHasNullsSelectionWithAggregationSelection( + aggregationBufferSets, aggregateIndex, + batchSize, batch.selected, inputVector.isNull); + } + } + + private void iterateNoNullsWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + int batchSize) { + + for (int i=0; i < batchSize; ++i) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + i); + myagg.initIfNull(); + myagg.value++; + } + } + + private void iterateHasNullsWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + int batchSize, + boolean[] isNull) { + + for (int i=0; i < batchSize; ++i) { + if (!isNull[i]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + i); + myagg.initIfNull(); + myagg.value++; + } + } + } + + private void iterateHasNullsSelectionWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + int batchSize, + int[] selection, + boolean[] isNull) { + + for (int j=0; j < batchSize; ++j) { + int i = selection[j]; + if (!isNull[i]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + j); + myagg.initIfNull(); + myagg.value++; + } + } + } + + + @Override + public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) + throws HiveException { + + inputExpression.evaluate(batch); + + inputVector = ()batch. + cols[this.inputExpression.getOutputColumn()]; + + int batchSize = batch.size; + + if (batchSize == 0) { + return; + } + + Aggregation myagg = (Aggregation)agg; + + myagg.initIfNull(); + + if (inputVector.isRepeating) { + if (inputVector.noNulls || !inputVector.isNull[0]) { + myagg.value += batchSize; + } + return; + } + + if (inputVector.noNulls) { + myagg.value += batchSize; + return; + } + else if (!batch.selectedInUse) { + iterateNoSelectionHasNulls(myagg, batchSize, inputVector.isNull); + } + else { + iterateSelectionHasNulls(myagg, batchSize, inputVector.isNull, batch.selected); + } + } + + private void iterateSelectionHasNulls( + Aggregation myagg, + int batchSize, + boolean[] isNull, + int[] selected) { + + for (int j=0; j< batchSize; ++j) { + int i = selected[j]; + if (!isNull[i]) { + myagg.value += 1; + } + } + } + + private void iterateNoSelectionHasNulls( + Aggregation myagg, + int batchSize, + boolean[] isNull) { + + for (int i=0; i< batchSize; ++i) { + if (!isNull[i]) { + myagg.value += 1; + } + } + } + + @Override + public AggregationBuffer getNewAggregationBuffer() throws HiveException { + return new Aggregation(); + } + + @Override + public void reset(AggregationBuffer agg) throws HiveException { + Aggregation myAgg = (Aggregation) agg; + myAgg.isNull = true; + } + + @Override + public Object evaluateOutput(AggregationBuffer agg) throws HiveException { + Aggregation myagg = (Aggregation) agg; + if (myagg.isNull) { + return null; + } + else { + result.set (myagg.value); + return result; + } + } + + @Override + public ObjectInspector getOutputObjectInspector() { + return PrimitiveObjectInspectorFactory.writableLongObjectInspector; + } + +} + diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFMinMax.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFMinMax.txt new file mode 100644 index 0000000..d00d9ae --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFMinMax.txt @@ -0,0 +1,415 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen; + +import java.util.ArrayList; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates. + VectorAggregateExpression.AggregationBuffer; +import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + +/** +* . Vectorized implementation for MIN/MAX aggregates. +*/ +@Description(name = "", value = "") +public class extends VectorAggregateExpression { + + /** + /* class for storing the current aggregate value. + */ + static private final class Aggregation implements AggregationBuffer { + value; + boolean isNull; + + public void checkValue( value) { + if (isNull) { + isNull = false; + this.value = value; + } else if (value this.value) { + this.value = value; + } + } + } + + private VectorExpression inputExpression; + private result; + + public (VectorExpression inputExpression) { + super(); + this.inputExpression = inputExpression; + result = new (); + } + + private Aggregation getCurrentAggregationBuffer( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregrateIndex, + int row) { + VectorAggregationBufferRow mySet = aggregationBufferSets[row]; + Aggregation myagg = (Aggregation) mySet.getAggregationBuffer(aggregrateIndex); + return myagg; + } + +@Override + public void aggregateInputSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregrateIndex, + VectorizedRowBatch batch) throws HiveException { + + int batchSize = batch.size; + + if (batchSize == 0) { + return; + } + + inputExpression.evaluate(batch); + + inputVector = ()batch. + cols[this.inputExpression.getOutputColumn()]; + [] vector = inputVector.vector; + + if (inputVector.noNulls) { + if (inputVector.isRepeating) { + iterateNoNullsRepeatingWithAggregationSelection( + aggregationBufferSets, aggregrateIndex, + vector[0], batchSize); + } else { + if (batch.selectedInUse) { + iterateNoNullsSelectionWithAggregationSelection( + aggregationBufferSets, aggregrateIndex, + vector, batch.selected, batchSize); + } else { + iterateNoNullsWithAggregationSelection( + aggregationBufferSets, aggregrateIndex, + vector, batchSize); + } + } + } else { + if (inputVector.isRepeating) { + if (batch.selectedInUse) { + iterateHasNullsRepeatingSelectionWithAggregationSelection( + aggregationBufferSets, aggregrateIndex, + vector[0], batchSize, batch.selected, inputVector.isNull); + } else { + iterateHasNullsRepeatingWithAggregationSelection( + aggregationBufferSets, aggregrateIndex, + vector[0], batchSize, inputVector.isNull); + } + } else { + if (batch.selectedInUse) { + iterateHasNullsSelectionWithAggregationSelection( + aggregationBufferSets, aggregrateIndex, + vector, batchSize, batch.selected, inputVector.isNull); + } else { + iterateHasNullsWithAggregationSelection( + aggregationBufferSets, aggregrateIndex, + vector, batchSize, inputVector.isNull); + } + } + } + } + + private void iterateNoNullsRepeatingWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregrateIndex, + value, + int batchSize) { + + for (int i=0; i < batchSize; ++i) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregrateIndex, + i); + myagg.checkValue(value); + } + } + + private void iterateNoNullsSelectionWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregrateIndex, + [] values, + int[] selection, + int batchSize) { + + for (int i=0; i < batchSize; ++i) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregrateIndex, + i); + myagg.checkValue(values[selection[i]]); + } + } + + private void iterateNoNullsWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregrateIndex, + [] values, + int batchSize) { + for (int i=0; i < batchSize; ++i) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregrateIndex, + i); + myagg.checkValue(values[i]); + } + } + + private void iterateHasNullsRepeatingSelectionWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregrateIndex, + value, + int batchSize, + int[] selection, + boolean[] isNull) { + + for (int i=0; i < batchSize; ++i) { + if (!isNull[selection[i]]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregrateIndex, + i); + myagg.checkValue(value); + } + } + + } + + private void iterateHasNullsRepeatingWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregrateIndex, + value, + int batchSize, + boolean[] isNull) { + + for (int i=0; i < batchSize; ++i) { + if (!isNull[i]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregrateIndex, + i); + myagg.checkValue(value); + } + } + } + + private void iterateHasNullsSelectionWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregrateIndex, + [] values, + int batchSize, + int[] selection, + boolean[] isNull) { + + for (int j=0; j < batchSize; ++j) { + int i = selection[j]; + if (!isNull[i]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregrateIndex, + j); + myagg.checkValue(values[i]); + } + } + } + + private void iterateHasNullsWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregrateIndex, + [] values, + int batchSize, + boolean[] isNull) { + + for (int i=0; i < batchSize; ++i) { + if (!isNull[i]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregrateIndex, + i); + myagg.checkValue(values[i]); + } + } + } + + @Override + public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) + throws HiveException { + + inputExpression.evaluate(batch); + + inputVector = ()batch. + cols[this.inputExpression.getOutputColumn()]; + + int batchSize = batch.size; + + if (batchSize == 0) { + return; + } + + Aggregation myagg = (Aggregation)agg; + + [] vector = inputVector.vector; + + if (inputVector.isRepeating) { + if ((inputVector.noNulls || !inputVector.isNull[0]) && + myagg.isNull || vector[0] < myagg.value) { + myagg.isNull = false; + myagg.value = vector[0]; + } + return; + } + + if (!batch.selectedInUse && inputVector.noNulls) { + iterateNoSelectionNoNulls(myagg, vector, batchSize); + } + else if (!batch.selectedInUse) { + iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull); + } + else if (inputVector.noNulls){ + iterateSelectionNoNulls(myagg, vector, batchSize, batch.selected); + } + else { + iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, batch.selected); + } + } + + private void iterateSelectionHasNulls( + Aggregation myagg, + [] vector, + int batchSize, + boolean[] isNull, + int[] selected) { + + for (int j=0; j< batchSize; ++j) { + int i = selected[j]; + if (!isNull[i]) { + value = vector[i]; + if (myagg.isNull) { + myagg.isNull = false; + myagg.value = value; + } + else if (value myagg.value) { + myagg.value = value; + } + } + } + } + + private void iterateSelectionNoNulls( + Aggregation myagg, + [] vector, + int batchSize, + int[] selected) { + + if (myagg.isNull) { + myagg.value = vector[selected[0]]; + myagg.isNull = false; + } + + for (int i=0; i< batchSize; ++i) { + value = vector[selected[i]]; + if (value myagg.value) { + myagg.value = value; + } + } + } + + private void iterateNoSelectionHasNulls( + Aggregation myagg, + [] vector, + int batchSize, + boolean[] isNull) { + + for(int i=0;i value = vector[i]; + if (myagg.isNull) { + myagg.value = value; + myagg.isNull = false; + } + else if (value myagg.value) { + myagg.value = value; + } + } + } + } + + private void iterateNoSelectionNoNulls( + Aggregation myagg, + [] vector, + int batchSize) { + if (myagg.isNull) { + myagg.value = vector[0]; + myagg.isNull = false; + } + + for (int i=0;i value = vector[i]; + if (value myagg.value) { + myagg.value = value; + } + } + } + + @Override + public AggregationBuffer getNewAggregationBuffer() throws HiveException { + return new Aggregation(); + } + + @Override + public void reset(AggregationBuffer agg) throws HiveException { + Aggregation myAgg = (Aggregation) agg; + myAgg.isNull = true; + } + + @Override + public Object evaluateOutput( + AggregationBuffer agg) throws HiveException { + Aggregation myagg = (Aggregation) agg; + if (myagg.isNull) { + return null; + } + else { + result.set(myagg.value); + return result; + } + } + + @Override + public ObjectInspector getOutputObjectInspector() { + return ; + } +} + diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFSum.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFSum.txt new file mode 100644 index 0000000..aaaa6ad --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFSum.txt @@ -0,0 +1,409 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen; + +import java.util.ArrayList; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates. + VectorAggregateExpression.AggregationBuffer; +import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + +/** +* . Vectorized implementation for SUM aggregates. +*/ +@Description(name = "sum", value = "_FUNC_(expr) - Returns the sum value of expr (vectorized, type: )") +public class extends VectorAggregateExpression { + + /** + /* class for storing the current aggregate value. + */ + private static final class Aggregation implements AggregationBuffer { + sum; + boolean isNull; + + public void sumValue( value) { + if (isNull) { + sum = value; + isNull = false; + } else { + sum += value; + } + } + } + + VectorExpression inputExpression; + private result; + + public (VectorExpression inputExpression) { + super(); + this.inputExpression = inputExpression; + result = new (); + } + + private Aggregation getCurrentAggregationBuffer( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + int row) { + VectorAggregationBufferRow mySet = aggregationBufferSets[row]; + Aggregation myagg = (Aggregation) mySet.getAggregationBuffer(aggregateIndex); + return myagg; + } + + @Override + public void aggregateInputSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + VectorizedRowBatch batch) throws HiveException { + + int batchSize = batch.size; + + if (batchSize == 0) { + return; + } + + inputExpression.evaluate(batch); + + LongColumnVector inputVector = (LongColumnVector)batch. + cols[this.inputExpression.getOutputColumn()]; + long[] vector = inputVector.vector; + + if (inputVector.noNulls) { + if (inputVector.isRepeating) { + iterateNoNullsRepeatingWithAggregationSelection( + aggregationBufferSets, aggregateIndex, + vector[0], batchSize); + } else { + if (batch.selectedInUse) { + iterateNoNullsSelectionWithAggregationSelection( + aggregationBufferSets, aggregateIndex, + vector, batch.selected, batchSize); + } else { + iterateNoNullsWithAggregationSelection( + aggregationBufferSets, aggregateIndex, + vector, batchSize); + } + } + } else { + if (inputVector.isRepeating) { + if (batch.selectedInUse) { + iterateHasNullsRepeatingSelectionWithAggregationSelection( + aggregationBufferSets, aggregateIndex, + vector[0], batchSize, batch.selected, inputVector.isNull); + } else { + iterateHasNullsRepeatingWithAggregationSelection( + aggregationBufferSets, aggregateIndex, + vector[0], batchSize, inputVector.isNull); + } + } else { + if (batch.selectedInUse) { + iterateHasNullsSelectionWithAggregationSelection( + aggregationBufferSets, aggregateIndex, + vector, batchSize, batch.selected, inputVector.isNull); + } else { + iterateHasNullsWithAggregationSelection( + aggregationBufferSets, aggregateIndex, + vector, batchSize, inputVector.isNull); + } + } + } + } + + private void iterateNoNullsRepeatingWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + long value, + int batchSize) { + + for (int i=0; i < batchSize; ++i) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + i); + myagg.sumValue(value); + } + } + + private void iterateNoNullsSelectionWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + long[] values, + int[] selection, + int batchSize) { + + for (int i=0; i < batchSize; ++i) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + i); + myagg.sumValue(values[selection[i]]); + } + } + + private void iterateNoNullsWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + long[] values, + int batchSize) { + for (int i=0; i < batchSize; ++i) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + i); + myagg.sumValue(values[i]); + } + } + + private void iterateHasNullsRepeatingSelectionWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + long value, + int batchSize, + int[] selection, + boolean[] isNull) { + + for (int i=0; i < batchSize; ++i) { + if (!isNull[selection[i]]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + i); + myagg.sumValue(value); + } + } + + } + + private void iterateHasNullsRepeatingWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + long value, + int batchSize, + boolean[] isNull) { + + for (int i=0; i < batchSize; ++i) { + if (!isNull[i]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + i); + myagg.sumValue(value); + } + } + } + + private void iterateHasNullsSelectionWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + long[] values, + int batchSize, + int[] selection, + boolean[] isNull) { + + for (int j=0; j < batchSize; ++j) { + int i = selection[j]; + if (!isNull[i]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + j); + myagg.sumValue(values[i]); + } + } + } + + private void iterateHasNullsWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + long[] values, + int batchSize, + boolean[] isNull) { + + for (int i=0; i < batchSize; ++i) { + if (!isNull[i]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + i); + myagg.sumValue(values[i]); + } + } + } + + + @Override + public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) + throws HiveException { + + inputExpression.evaluate(batch); + + inputVector = ()batch. + cols[this.inputExpression.getOutputColumn()]; + + int batchSize = batch.size; + + if (batchSize == 0) { + return; + } + + Aggregation myagg = (Aggregation)agg; + + [] vector = inputVector.vector; + + if (inputVector.isRepeating) { + if (inputVector.noNulls || !inputVector.isNull[0]) { + if (myagg.isNull) { + myagg.isNull = false; + myagg.sum = 0; + } + myagg.sum += vector[0]*batchSize; + } + return; + } + + if (!batch.selectedInUse && inputVector.noNulls) { + iterateNoSelectionNoNulls(myagg, vector, batchSize); + } + else if (!batch.selectedInUse) { + iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull); + } + else if (inputVector.noNulls){ + iterateSelectionNoNulls(myagg, vector, batchSize, batch.selected); + } + else { + iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, batch.selected); + } + } + + private void iterateSelectionHasNulls( + Aggregation myagg, + [] vector, + int batchSize, + boolean[] isNull, + int[] selected) { + + for (int j=0; j< batchSize; ++j) { + int i = selected[j]; + if (!isNull[i]) { + value = vector[i]; + if (myagg.isNull) { + myagg.isNull = false; + myagg.sum = 0; + } + myagg.sum += value; + } + } + } + + private void iterateSelectionNoNulls( + Aggregation myagg, + [] vector, + int batchSize, + int[] selected) { + + if (myagg.isNull) { + myagg.sum = 0; + myagg.isNull = false; + } + + for (int i=0; i< batchSize; ++i) { + value = vector[selected[i]]; + myagg.sum += value; + } + } + + private void iterateNoSelectionHasNulls( + Aggregation myagg, + [] vector, + int batchSize, + boolean[] isNull) { + + for(int i=0;i value = vector[i]; + if (myagg.isNull) { + myagg.sum = 0; + myagg.isNull = false; + } + myagg.sum += value; + } + } + } + + private void iterateNoSelectionNoNulls( + Aggregation myagg, + [] vector, + int batchSize) { + if (myagg.isNull) { + myagg.sum = 0; + myagg.isNull = false; + } + + for (int i=0;i value = vector[i]; + myagg.sum += value; + } + } + + @Override + public AggregationBuffer getNewAggregationBuffer() throws HiveException { + return new Aggregation(); + } + + @Override + public void reset(AggregationBuffer agg) throws HiveException { + Aggregation myAgg = (Aggregation) agg; + myAgg.isNull = true; + } + + @Override + public Object evaluateOutput(AggregationBuffer agg) throws HiveException { + Aggregation myagg = (Aggregation) agg; + if (myagg.isNull) { + return null; + } + else { + result.set(myagg.sum); + return result; + } + } + + @Override + public ObjectInspector getOutputObjectInspector() { + return ; + } +} + diff --git ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFVar.txt ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFVar.txt new file mode 100644 index 0000000..daae57b --- /dev/null +++ ql/src/gen/vectorization/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/VectorUDAFVar.txt @@ -0,0 +1,487 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen; + +import java.util.ArrayList; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates + .VectorAggregateExpression.AggregationBuffer; +import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + +/** +* . Vectorized implementation for VARIANCE aggregates. +*/ +@Description(name = "", value = "") +public class extends VectorAggregateExpression { + + /** + /* class for storing the current aggregate value. + */ + private static final class Aggregation implements AggregationBuffer { + double sum; + long count; + double variance; + boolean isNull; + + public void init() { + isNull = false; + sum = 0; + count = 0; + variance = 0; + } + } + + private VectorExpression inputExpression; + private LongWritable resultCount; + private DoubleWritable resultSum; + private DoubleWritable resultVariance; + private Object[] partialResult; + + private ObjectInspector soi; + + + public (VectorExpression inputExpression) { + super(); + this.inputExpression = inputExpression; + partialResult = new Object[3]; + resultCount = new LongWritable(); + resultSum = new DoubleWritable(); + resultVariance = new DoubleWritable(); + partialResult[0] = resultCount; + partialResult[1] = resultSum; + partialResult[2] = resultVariance; + initPartialResultInspector(); + } + + private void initPartialResultInspector() { + ArrayList foi = new ArrayList(); + foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector); + foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); + foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); + + ArrayList fname = new ArrayList(); + fname.add("count"); + fname.add("sum"); + fname.add("variance"); + + soi = ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi); + } + + private Aggregation getCurrentAggregationBuffer( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + int row) { + VectorAggregationBufferRow mySet = aggregationBufferSets[row]; + Aggregation myagg = (Aggregation) mySet.getAggregationBuffer(aggregateIndex); + return myagg; + } + + + @Override + public void aggregateInputSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + VectorizedRowBatch batch) throws HiveException { + + inputExpression.evaluate(batch); + + inputVector = ()batch. + cols[this.inputExpression.getOutputColumn()]; + + int batchSize = batch.size; + + if (batchSize == 0) { + return; + } + + [] vector = inputVector.vector; + + if (inputVector.isRepeating) { + if (inputVector.noNulls || !inputVector.isNull[0]) { + iterateRepeatingNoNullsWithAggregationSelection( + aggregationBufferSets, aggregateIndex, vector[0], batchSize); + } + } + else if (!batch.selectedInUse && inputVector.noNulls) { + iterateNoSelectionNoNullsWithAggregationSelection( + aggregationBufferSets, aggregateIndex, vector, batchSize); + } + else if (!batch.selectedInUse) { + iterateNoSelectionHasNullsWithAggregationSelection( + aggregationBufferSets, aggregateIndex, vector, batchSize, inputVector.isNull); + } + else if (inputVector.noNulls){ + iterateSelectionNoNullsWithAggregationSelection( + aggregationBufferSets, aggregateIndex, vector, batchSize, batch.selected); + } + else { + iterateSelectionHasNullsWithAggregationSelection( + aggregationBufferSets, aggregateIndex, vector, batchSize, + inputVector.isNull, batch.selected); + } + + } + + private void iterateRepeatingNoNullsWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + value, + int batchSize) { + + for (int i=0; i 1) { + double t = myagg.count*value - myagg.sum; + myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); + } + } + } + + private void iterateSelectionHasNullsWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + [] vector, + int batchSize, + boolean[] isNull, + int[] selected) { + + for (int j=0; j< batchSize; ++j) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + j); + int i = selected[j]; + if (!isNull[i]) { + value = vector[i]; + if (myagg.isNull) { + myagg.init (); + } + myagg.sum += value; + myagg.count += 1; + if(myagg.count > 1) { + double t = myagg.count*value - myagg.sum; + myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); + } + } + } + } + + private void iterateSelectionNoNullsWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + [] vector, + int batchSize, + int[] selected) { + + for (int i=0; i< batchSize; ++i) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + i); + value = vector[selected[i]]; + if (myagg.isNull) { + myagg.init (); + } + myagg.sum += value; + myagg.count += 1; + if(myagg.count > 1) { + double t = myagg.count*value - myagg.sum; + myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); + } + } + } + + private void iterateNoSelectionHasNullsWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + [] vector, + int batchSize, + boolean[] isNull) { + + for(int i=0;i value = vector[i]; + if (myagg.isNull) { + myagg.init (); + } + myagg.sum += value; + myagg.count += 1; + if(myagg.count > 1) { + double t = myagg.count*value - myagg.sum; + myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); + } + } + } + } + + private void iterateNoSelectionNoNullsWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + [] vector, + int batchSize) { + + for (int i=0; i value = vector[i]; + myagg.sum += value; + myagg.count += 1; + if(myagg.count > 1) { + double t = myagg.count*value - myagg.sum; + myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); + } + } + } + + @Override + public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) + throws HiveException { + + inputExpression.evaluate(batch); + + inputVector = ()batch. + cols[this.inputExpression.getOutputColumn()]; + + int batchSize = batch.size; + + if (batchSize == 0) { + return; + } + + Aggregation myagg = (Aggregation)agg; + + [] vector = inputVector.vector; + + if (inputVector.isRepeating) { + if (inputVector.noNulls || !inputVector.isNull[0]) { + iterateRepeatingNoNulls(myagg, vector[0], batchSize); + } + } + else if (!batch.selectedInUse && inputVector.noNulls) { + iterateNoSelectionNoNulls(myagg, vector, batchSize); + } + else if (!batch.selectedInUse) { + iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull); + } + else if (inputVector.noNulls){ + iterateSelectionNoNulls(myagg, vector, batchSize, batch.selected); + } + else { + iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, batch.selected); + } + } + + private void iterateRepeatingNoNulls( + Aggregation myagg, + value, + int batchSize) { + + if (myagg.isNull) { + myagg.init (); + } + + // TODO: conjure a formula w/o iterating + // + + myagg.sum += value; + myagg.count += 1; + if(myagg.count > 1) { + double t = myagg.count*value - myagg.sum; + myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); + } + + // We pulled out i=0 so we can remove the count > 1 check in the loop + for (int i=1; i[] vector, + int batchSize, + boolean[] isNull, + int[] selected) { + + for (int j=0; j< batchSize; ++j) { + int i = selected[j]; + if (!isNull[i]) { + value = vector[i]; + if (myagg.isNull) { + myagg.init (); + } + myagg.sum += value; + myagg.count += 1; + if(myagg.count > 1) { + double t = myagg.count*value - myagg.sum; + myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); + } + } + } + } + + private void iterateSelectionNoNulls( + Aggregation myagg, + [] vector, + int batchSize, + int[] selected) { + + if (myagg.isNull) { + myagg.init (); + } + + value = vector[selected[0]]; + myagg.sum += value; + myagg.count += 1; + if(myagg.count > 1) { + double t = myagg.count*value - myagg.sum; + myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); + } + + // i=0 was pulled out to remove the count > 1 check in the loop + // + for (int i=1; i< batchSize; ++i) { + value = vector[selected[i]]; + myagg.sum += value; + myagg.count += 1; + double t = myagg.count*value - myagg.sum; + myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); + } + } + + private void iterateNoSelectionHasNulls( + Aggregation myagg, + [] vector, + int batchSize, + boolean[] isNull) { + + for(int i=0;i value = vector[i]; + if (myagg.isNull) { + myagg.init (); + } + myagg.sum += value; + myagg.count += 1; + if(myagg.count > 1) { + double t = myagg.count*value - myagg.sum; + myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); + } + } + } + } + + private void iterateNoSelectionNoNulls( + Aggregation myagg, + [] vector, + int batchSize) { + + if (myagg.isNull) { + myagg.init (); + } + + value = vector[0]; + myagg.sum += value; + myagg.count += 1; + + if(myagg.count > 1) { + double t = myagg.count*value - myagg.sum; + myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); + } + + // i=0 was pulled out to remove count > 1 check + for (int i=1; i"}, - {"ColumnCompareScalar", "Greater", "double", "double", ">"}, - {"ColumnCompareScalar", "GreaterEqual", "long", "double", ">="}, - {"ColumnCompareScalar", "GreaterEqual", "double", "double", ">="}, - - {"FilterColumnCompareScalar", "Equal", "long", "double", "=="}, - {"FilterColumnCompareScalar", "Equal", "double", "double", "=="}, - {"FilterColumnCompareScalar", "NotEqual", "long", "double", "!="}, - {"FilterColumnCompareScalar", "NotEqual", "double", "double", "!="}, - {"FilterColumnCompareScalar", "Less", "long", "double", "<"}, - {"FilterColumnCompareScalar", "Less", "double", "double", "<"}, - {"FilterColumnCompareScalar", "LessEqual", "long", "double", "<="}, - {"FilterColumnCompareScalar", "LessEqual", "double", "double", "<="}, - {"FilterColumnCompareScalar", "Greater", "long", "double", ">"}, - {"FilterColumnCompareScalar", "Greater", "double", "double", ">"}, - {"FilterColumnCompareScalar", "GreaterEqual", "long", "double", ">="}, - {"FilterColumnCompareScalar", "GreaterEqual", "double", "double", ">="}, - - {"FilterColumnCompareScalar", "Equal", "long", "long", "=="}, - {"FilterColumnCompareScalar", "Equal", "double", "long", "=="}, - {"FilterColumnCompareScalar", "NotEqual", "long", "long", "!="}, - {"FilterColumnCompareScalar", "NotEqual", "double", "long", "!="}, - {"FilterColumnCompareScalar", "Less", "long", "long", "<"}, - {"FilterColumnCompareScalar", "Less", "double", "long", "<"}, - {"FilterColumnCompareScalar", "LessEqual", "long", "long", "<="}, - {"FilterColumnCompareScalar", "LessEqual", "double", "long", "<="}, - {"FilterColumnCompareScalar", "Greater", "long", "long", ">"}, - {"FilterColumnCompareScalar", "Greater", "double", "long", ">"}, - {"FilterColumnCompareScalar", "GreaterEqual", "long", "long", ">="}, - {"FilterColumnCompareScalar", "GreaterEqual", "double", "long", ">="}, - - {"FilterScalarCompareColumn", "Equal", "long", "double", "=="}, - {"FilterScalarCompareColumn", "Equal", "double", "double", "=="}, - {"FilterScalarCompareColumn", "NotEqual", "long", "double", "!="}, - {"FilterScalarCompareColumn", "NotEqual", "double", "double", "!="}, - {"FilterScalarCompareColumn", "Less", "long", "double", "<"}, - {"FilterScalarCompareColumn", "Less", "double", "double", "<"}, - {"FilterScalarCompareColumn", "LessEqual", "long", "double", "<="}, - {"FilterScalarCompareColumn", "LessEqual", "double", "double", "<="}, - {"FilterScalarCompareColumn", "Greater", "long", "double", ">"}, - {"FilterScalarCompareColumn", "Greater", "double", "double", ">"}, - {"FilterScalarCompareColumn", "GreaterEqual", "long", "double", ">="}, - {"FilterScalarCompareColumn", "GreaterEqual", "double", "double", ">="}, - - {"FilterScalarCompareColumn", "Equal", "long", "long", "=="}, - {"FilterScalarCompareColumn", "Equal", "double", "long", "=="}, - {"FilterScalarCompareColumn", "NotEqual", "long", "long", "!="}, - {"FilterScalarCompareColumn", "NotEqual", "double", "long", "!="}, - {"FilterScalarCompareColumn", "Less", "long", "long", "<"}, - {"FilterScalarCompareColumn", "Less", "double", "long", "<"}, - {"FilterScalarCompareColumn", "LessEqual", "long", "long", "<="}, - {"FilterScalarCompareColumn", "LessEqual", "double", "long", "<="}, - {"FilterScalarCompareColumn", "Greater", "long", "long", ">"}, - {"FilterScalarCompareColumn", "Greater", "double", "long", ">"}, - {"FilterScalarCompareColumn", "GreaterEqual", "long", "long", ">="}, - {"FilterScalarCompareColumn", "GreaterEqual", "double", "long", ">="}, - - {"FilterStringColumnCompareScalar", "Equal", "=="}, - {"FilterStringColumnCompareScalar", "NotEqual", "!="}, - {"FilterStringColumnCompareScalar", "Less", "<"}, - {"FilterStringColumnCompareScalar", "LessEqual", "<="}, - {"FilterStringColumnCompareScalar", "Greater", ">"}, - {"FilterStringColumnCompareScalar", "GreaterEqual", ">="}, - - {"FilterStringColumnCompareColumn", "Equal", "=="}, - {"FilterStringColumnCompareColumn", "NotEqual", "!="}, - {"FilterStringColumnCompareColumn", "Less", "<"}, - {"FilterStringColumnCompareColumn", "LessEqual", "<="}, - {"FilterStringColumnCompareColumn", "Greater", ">"}, - {"FilterStringColumnCompareColumn", "GreaterEqual", ">="}, - - {"FilterColumnCompareColumn", "Equal", "long", "double", "=="}, - {"FilterColumnCompareColumn", "Equal", "double", "double", "=="}, - {"FilterColumnCompareColumn", "NotEqual", "long", "double", "!="}, - {"FilterColumnCompareColumn", "NotEqual", "double", "double", "!="}, - {"FilterColumnCompareColumn", "Less", "long", "double", "<"}, - {"FilterColumnCompareColumn", "Less", "double", "double", "<"}, - {"FilterColumnCompareColumn", "LessEqual", "long", "double", "<="}, - {"FilterColumnCompareColumn", "LessEqual", "double", "double", "<="}, - {"FilterColumnCompareColumn", "Greater", "long", "double", ">"}, - {"FilterColumnCompareColumn", "Greater", "double", "double", ">"}, - {"FilterColumnCompareColumn", "GreaterEqual", "long", "double", ">="}, - {"FilterColumnCompareColumn", "GreaterEqual", "double", "double", ">="}, - - {"FilterColumnCompareColumn", "Equal", "long", "long", "=="}, - {"FilterColumnCompareColumn", "Equal", "double", "long", "=="}, - {"FilterColumnCompareColumn", "NotEqual", "long", "long", "!="}, - {"FilterColumnCompareColumn", "NotEqual", "double", "long", "!="}, - {"FilterColumnCompareColumn", "Less", "long", "long", "<"}, - {"FilterColumnCompareColumn", "Less", "double", "long", "<"}, - {"FilterColumnCompareColumn", "LessEqual", "long", "long", "<="}, - {"FilterColumnCompareColumn", "LessEqual", "double", "long", "<="}, - {"FilterColumnCompareColumn", "Greater", "long", "long", ">"}, - {"FilterColumnCompareColumn", "Greater", "double", "long", ">"}, - {"FilterColumnCompareColumn", "GreaterEqual", "long", "long", ">="}, - {"FilterColumnCompareColumn", "GreaterEqual", "double", "long", ">="}, - - {"ColumnUnaryMinus", "long"}, - {"ColumnUnaryMinus", "double"}, - - //template, , , , , - {"VectorUDAFMinMax", "VectorUDAFMinLong", "long", "<", "min", "_FUNC_(expr) - Returns the minimum value of expr (vectorized, type: long)"}, - {"VectorUDAFMinMax", "VectorUDAFMinDouble", "double", "<", "min", "_FUNC_(expr) - Returns the minimum value of expr (vectorized, type: double)"}, - {"VectorUDAFMinMax", "VectorUDAFMaxLong", "long", ">", "max", "_FUNC_(expr) - Returns the maximum value of expr (vectorized, type: long)"}, - {"VectorUDAFMinMax", "VectorUDAFMaxDouble", "double", ">", "max", "_FUNC_(expr) - Returns the maximum value of expr (vectorized, type: double)"}, - - //template, , - {"VectorUDAFCount", "VectorUDAFCountLong", "long"}, - {"VectorUDAFCount", "VectorUDAFCountDouble", "double"}, - - //template, , - {"VectorUDAFSum", "VectorUDAFSumLong", "long"}, - {"VectorUDAFSum", "VectorUDAFSumDouble", "double"}, - {"VectorUDAFAvg", "VectorUDAFAvgLong", "long"}, - {"VectorUDAFAvg", "VectorUDAFAvgDouble", "double"}, - - //template, , , , , - {"VectorUDAFVar", "VectorUDAFVarPopLong", "long", "myagg.variance / myagg.count", "variance, var_pop", "_FUNC_(x) - Returns the variance of a set of numbers (vectorized, long)"}, - {"VectorUDAFVar", "VectorUDAFVarPopDouble", "double", "myagg.variance / myagg.count", "variance, var_pop", "_FUNC_(x) - Returns the variance of a set of numbers (vectorized, double)"}, - {"VectorUDAFVar", "VectorUDAFVarSampLong", "long", "myagg.variance / (myagg.count-1.0)", "var_samp", "_FUNC_(x) - Returns the sample variance of a set of numbers (vectorized, long)"}, - {"VectorUDAFVar", "VectorUDAFVarSampDouble", "double", "myagg.variance / (myagg.count-1.0)", "var_samp", "_FUNC_(x) - Returns the sample variance of a set of numbers (vectorized, double)"}, - {"VectorUDAFVar", "VectorUDAFStdPopLong", "long", "Math.sqrt(myagg.variance / (myagg.count))", "std,stddev,stddev_pop", "_FUNC_(x) - Returns the standard deviation of a set of numbers (vectorized, long)"}, - {"VectorUDAFVar", "VectorUDAFStdPopDouble", "double", "Math.sqrt(myagg.variance / (myagg.count))", "std,stddev,stddev_pop", "_FUNC_(x) - Returns the standard deviation of a set of numbers (vectorized, double)"}, - {"VectorUDAFVar", "VectorUDAFStdSampLong", "long", "Math.sqrt(myagg.variance / (myagg.count-1.0))", "stddev_samp", "_FUNC_(x) - Returns the sample standard deviation of a set of numbers (vectorized, long)"}, - {"VectorUDAFVar", "VectorUDAFStdSampDouble", "double", "Math.sqrt(myagg.variance / (myagg.count-1.0))", "stddev_samp", "_FUNC_(x) - Returns the sample standard deviation of a set of numbers (vectorized, double)"}, - - }; - - - private final String templateDirectory; - private final String outputDirectory; - private final TestCodeGen testCodeGen; - - static String joinPath(String...parts) { - String path = parts[0]; - for (int i=1; i < parts.length; ++i) { - path += File.separatorChar + parts[i]; - } - return path; - } - - public CodeGen() { - templateDirectory = System.getProperty("user.dir"); - File f = new File(templateDirectory); - outputDirectory = joinPath(f.getParent(), "gen"); - testCodeGen = new TestCodeGen(joinPath(f.getParent(), "test"),templateDirectory); - } - - public CodeGen(String templateDirectory, String outputDirectory, String testOutputDirectory) { - this.templateDirectory = templateDirectory; - this.outputDirectory = outputDirectory; - testCodeGen = new TestCodeGen(testOutputDirectory,templateDirectory); - } - - /** - * @param args - * @throws Exception - */ - public static void main(String[] args) throws Exception { - CodeGen gen; - if (args == null || args.length==0) { - gen = new CodeGen(); - } else if (args.length==3) { - gen = new CodeGen(args[0], args[1], args[2]); - }else{ - System.out.println("args: "); - return; - } - gen.generate(); - } - - private void generate() throws Exception { - - for (String [] tdesc : templateExpansions) { - if (tdesc[0].equals("ColumnArithmeticScalar")) { - generateColumnArithmeticScalar(tdesc); - } else if (tdesc[0].equals("ColumnCompareScalar")) { - generateColumnCompareScalar(tdesc); - } else if (tdesc[0].equals("FilterColumnCompareScalar")) { - generateFilterColumnCompareScalar(tdesc); - } else if (tdesc[0].equals("ScalarArithmeticColumn")) { - generateScalarArithmeticColumn(tdesc); - } else if (tdesc[0].equals("FilterColumnCompareColumn")) { - generateFilterColumnCompareColumn(tdesc); - } else if (tdesc[0].equals("ColumnArithmeticColumn")) { - generateColumnArithmeticColumn(tdesc); - } else if (tdesc[0].equals("ColumnUnaryMinus")) { - generateColumnUnaryMinus(tdesc); - } else if (tdesc[0].equals("VectorUDAFCount")) { - generateVectorUDAFCount(tdesc); - } else if (tdesc[0].equals("VectorUDAFMinMax")) { - generateVectorUDAFMinMax(tdesc); - } else if (tdesc[0].equals("VectorUDAFSum")) { - generateVectorUDAFSum(tdesc); - } else if (tdesc[0].equals("VectorUDAFAvg")) { - generateVectorUDAFAvg(tdesc); - } else if (tdesc[0].equals("VectorUDAFVar")) { - generateVectorUDAFVar(tdesc); - } else if (tdesc[0].equals("FilterStringColumnCompareScalar")) { - generateFilterStringColumnCompareScalar(tdesc); - } else if (tdesc[0].equals("FilterStringColumnCompareColumn")) { - generateFilterStringColumnCompareColumn(tdesc); - } else { - continue; - } - } - testCodeGen.generateTestSuites(); - } - - private void generateVectorUDAFMinMax(String[] tdesc) throws Exception { - String className = tdesc[1]; - String valueType = tdesc[2]; - String operatorSymbol = tdesc[3]; - String descName = tdesc[4]; - String descValue = tdesc[5]; - String columnType = getColumnVectorType(valueType); - String writableType = getOutputWritableType(valueType); - String inspectorType = getOutputObjectInspector(valueType); - - String outputFile = joinPath(this.outputDirectory, className + ".java"); - String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); - - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("", className); - templateString = templateString.replaceAll("", valueType); - templateString = templateString.replaceAll("", operatorSymbol); - templateString = templateString.replaceAll("", columnType); - templateString = templateString.replaceAll("", descName); - templateString = templateString.replaceAll("", descValue); - templateString = templateString.replaceAll("", writableType); - templateString = templateString.replaceAll("", inspectorType); - writeFile(outputFile, templateString); - - } - - private void generateVectorUDAFCount(String[] tdesc) throws IOException { - String className = tdesc[1]; - String valueType = tdesc[2]; - String columnType = getColumnVectorType(valueType); - - String outputFile = joinPath(this.outputDirectory, className + ".java"); - String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); - - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("", className); - templateString = templateString.replaceAll("", valueType); - templateString = templateString.replaceAll("", columnType); - writeFile(outputFile, templateString); - } - - private void generateVectorUDAFSum(String[] tdesc) throws Exception { - //template, , , , - String className = tdesc[1]; - String valueType = tdesc[2]; - String columnType = getColumnVectorType(valueType); - String writableType = getOutputWritableType(valueType); - String inspectorType = getOutputObjectInspector(valueType); - - String outputFile = joinPath(this.outputDirectory, className + ".java"); - String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); - - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("", className); - templateString = templateString.replaceAll("", valueType); - templateString = templateString.replaceAll("", columnType); - templateString = templateString.replaceAll("", writableType); - templateString = templateString.replaceAll("", inspectorType); - writeFile(outputFile, templateString); - } - - private void generateVectorUDAFAvg(String[] tdesc) throws IOException { - String className = tdesc[1]; - String valueType = tdesc[2]; - String columnType = getColumnVectorType(valueType); - - String outputFile = joinPath(this.outputDirectory, className + ".java"); - String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); - - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("", className); - templateString = templateString.replaceAll("", valueType); - templateString = templateString.replaceAll("", columnType); - writeFile(outputFile, templateString); - } - - private void generateVectorUDAFVar(String[] tdesc) throws IOException { - String className = tdesc[1]; - String valueType = tdesc[2]; - String varianceFormula = tdesc[3]; - String descriptionName = tdesc[4]; - String descriptionValue = tdesc[5]; - String columnType = getColumnVectorType(valueType); - - String outputFile = joinPath(this.outputDirectory, className + ".java"); - String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); - - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("", className); - templateString = templateString.replaceAll("", valueType); - templateString = templateString.replaceAll("", columnType); - templateString = templateString.replaceAll("", varianceFormula); - templateString = templateString.replaceAll("", descriptionName); - templateString = templateString.replaceAll("", descriptionValue); - writeFile(outputFile, templateString); - } - - - private void generateFilterStringColumnCompareScalar(String[] tdesc) throws IOException { - String operatorName = tdesc[1]; - String className = "FilterStringCol" + operatorName + "StringScalar"; - generateFilterStringColumnCompareScalar(tdesc,className); - } - - private void generateFilterStringColumnCompareColumn(String[] tdesc) throws IOException { - String operatorName = tdesc[1]; - String className = "FilterStringCol" + operatorName + "StringCol"; - generateFilterStringColumnCompareScalar(tdesc,className); - } - - private void generateFilterStringColumnCompareScalar(String[] tdesc, String className) throws IOException { - String operatorSymbol = tdesc[2]; - String outputFile = joinPath(this.outputDirectory, className + ".java"); - // Read the template into a string; - String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); - String templateString = readFile(templateFile); - // Expand, and write result - templateString = templateString.replaceAll("", className); - templateString = templateString.replaceAll("", operatorSymbol); - writeFile(outputFile, templateString); - } - - private void generateFilterColumnCompareColumn(String[] tdesc) throws IOException { - //The variables are all same as ColumnCompareScalar except that - //this template doesn't need a return type. Pass anything as return type. - String operatorName = tdesc[1]; - String operandType1 = tdesc[2]; - String operandType2 = tdesc[3]; - String className = "Filter" + getCamelCaseType(operandType1) - + "Col" + operatorName + getCamelCaseType(operandType2) + "Column"; - generateColumnBinaryOperatorColumn(tdesc, null, className); - } - - private void generateColumnUnaryMinus(String[] tdesc) throws IOException { - String operandType = tdesc[1]; - String inputColumnVectorType = this.getColumnVectorType(operandType); - String outputColumnVectorType = inputColumnVectorType; - String returnType = operandType; - String className = getCamelCaseType(operandType) + "ColUnaryMinus"; - String outputFile = joinPath(this.outputDirectory, className + ".java"); - String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); - String templateString = readFile(templateFile); - // Expand, and write result - templateString = templateString.replaceAll("", className); - templateString = templateString.replaceAll("", inputColumnVectorType); - templateString = templateString.replaceAll("", outputColumnVectorType); - templateString = templateString.replaceAll("", operandType); - templateString = templateString.replaceAll("", returnType); - writeFile(outputFile, templateString); - } - - private void generateColumnArithmeticColumn(String [] tdesc) throws IOException { - String operatorName = tdesc[1]; - String operandType1 = tdesc[2]; - String operandType2 = tdesc[3]; - String className = getCamelCaseType(operandType1) - + "Col" + operatorName + getCamelCaseType(operandType2) + "Column"; - String returnType = getArithmeticReturnType(operandType1, operandType2); - generateColumnBinaryOperatorColumn(tdesc, returnType, className); - } - - private void generateFilterColumnCompareScalar(String[] tdesc) throws IOException { - //The variables are all same as ColumnCompareScalar except that - //this template doesn't need a return type. Pass anything as return type. - String operatorName = tdesc[1]; - String operandType1 = tdesc[2]; - String operandType2 = tdesc[3]; - String className = "Filter" + getCamelCaseType(operandType1) - + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar"; - generateColumnBinaryOperatorScalar(tdesc, null, className); - } - - private void generateFilterScalarCompareColumn(String[] tdesc) throws IOException { - //this template doesn't need a return type. Pass anything as return type. - String operatorName = tdesc[1]; - String operandType1 = tdesc[2]; - String operandType2 = tdesc[3]; - String className = "Filter" + getCamelCaseType(operandType1) - + "Scalar" + operatorName + getCamelCaseType(operandType2) + "Column"; - generateScalarBinaryOperatorColumn(tdesc, null, className); - } - - private void generateColumnCompareScalar(String[] tdesc) throws IOException { - String operatorName = tdesc[1]; - String operandType1 = tdesc[2]; - String operandType2 = tdesc[3]; - String returnType = "long"; - String className = getCamelCaseType(operandType1) - + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar"; - generateColumnBinaryOperatorScalar(tdesc, returnType, className); - } - - private void generateColumnBinaryOperatorColumn(String[] tdesc, String returnType, - String className) throws IOException { - String operandType1 = tdesc[2]; - String operandType2 = tdesc[3]; - String outputColumnVectorType = this.getColumnVectorType(returnType); - String inputColumnVectorType1 = this.getColumnVectorType(operandType1); - String inputColumnVectorType2 = this.getColumnVectorType(operandType2); - String operatorSymbol = tdesc[4]; - String outputFile = joinPath(this.outputDirectory, className + ".java"); - - //Read the template into a string; - String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("", className); - templateString = templateString.replaceAll("", inputColumnVectorType1); - templateString = templateString.replaceAll("", inputColumnVectorType2); - templateString = templateString.replaceAll("", outputColumnVectorType); - templateString = templateString.replaceAll("", operatorSymbol); - templateString = templateString.replaceAll("", operandType1); - templateString = templateString.replaceAll("", operandType2); - templateString = templateString.replaceAll("", returnType); - writeFile(outputFile, templateString); - - if(returnType==null){ - testCodeGen.addColumnColumnFilterTestCases( - className, - inputColumnVectorType1, - inputColumnVectorType2, - operatorSymbol); - }else{ - testCodeGen.addColumnColumnOperationTestCases( - className, - inputColumnVectorType1, - inputColumnVectorType2, - outputColumnVectorType); - } - } - - private void generateColumnBinaryOperatorScalar(String[] tdesc, String returnType, - String className) throws IOException { - String operandType1 = tdesc[2]; - String operandType2 = tdesc[3]; - String outputColumnVectorType = this.getColumnVectorType(returnType); - String inputColumnVectorType = this.getColumnVectorType(operandType1); - String operatorSymbol = tdesc[4]; - String outputFile = joinPath(this.outputDirectory, className + ".java"); - - //Read the template into a string; - String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("", className); - templateString = templateString.replaceAll("", inputColumnVectorType); - templateString = templateString.replaceAll("", outputColumnVectorType); - templateString = templateString.replaceAll("", operatorSymbol); - templateString = templateString.replaceAll("", operandType1); - templateString = templateString.replaceAll("", operandType2); - templateString = templateString.replaceAll("", returnType); - writeFile(outputFile, templateString); - - if(returnType==null) - { - testCodeGen.addColumnScalarFilterTestCases( - true, - className, - inputColumnVectorType, - operandType2, - operatorSymbol); - }else - { - testCodeGen.addColumnScalarOperationTestCases( - true, - className, - inputColumnVectorType, - outputColumnVectorType, - operandType2); - } - - } - - private void generateScalarBinaryOperatorColumn(String[] tdesc, String returnType, - String className) throws IOException { - String operandType1 = tdesc[2]; - String operandType2 = tdesc[3]; - String outputColumnVectorType = this.getColumnVectorType(returnType); - String inputColumnVectorType = this.getColumnVectorType(operandType2); - String operatorSymbol = tdesc[4]; - String outputFile = joinPath(this.outputDirectory, className + ".java"); - - //Read the template into a string; - String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt"); - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("", className); - templateString = templateString.replaceAll("", inputColumnVectorType); - templateString = templateString.replaceAll("", outputColumnVectorType); - templateString = templateString.replaceAll("", operatorSymbol); - templateString = templateString.replaceAll("", operandType1); - templateString = templateString.replaceAll("", operandType2); - templateString = templateString.replaceAll("", returnType); - writeFile(outputFile, templateString); - - if(returnType==null) - { - testCodeGen.addColumnScalarFilterTestCases( - false, - className, - inputColumnVectorType, - operandType1, - operatorSymbol); - }else - { - testCodeGen.addColumnScalarOperationTestCases( - false, - className, - inputColumnVectorType, - outputColumnVectorType, - operandType1); - } - } - - //Binary arithmetic operator - private void generateColumnArithmeticScalar(String[] tdesc) throws IOException { - String operatorName = tdesc[1]; - String operandType1 = tdesc[2]; - String operandType2 = tdesc[3]; - String className = getCamelCaseType(operandType1) - + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar"; - String returnType = getArithmeticReturnType(operandType1, operandType2); - generateColumnBinaryOperatorScalar(tdesc, returnType, className); - } - - private void generateScalarArithmeticColumn(String[] tdesc) throws IOException { - String operatorName = tdesc[1]; - String operandType1 = tdesc[2]; - String operandType2 = tdesc[3]; - String className = getCamelCaseType(operandType1) - + "Scalar" + operatorName + getCamelCaseType(operandType2) + "Column"; - String returnType = getArithmeticReturnType(operandType1, operandType2); - generateScalarBinaryOperatorColumn(tdesc, returnType, className); - } - - - static void writeFile(String outputFile, String str) throws IOException { - BufferedWriter w = new BufferedWriter(new FileWriter(outputFile)); - w.write(str); - w.close(); - } - - static String readFile(String templateFile) throws IOException { - BufferedReader r = new BufferedReader(new FileReader(templateFile)); - String line = r.readLine(); - StringBuilder b = new StringBuilder(); - while (line != null) { - b.append(line); - b.append("\n"); - line = r.readLine(); - } - r.close(); - return b.toString(); - } - - static String getCamelCaseType(String type) { - if (type.equals("long")) { - return "Long"; - } else if (type.equals("double")) { - return "Double"; - } else { - return type; - } - } - - private String getArithmeticReturnType(String operandType1, - String operandType2) { - if (operandType1.equals("double") || - operandType2.equals("double")) { - return "double"; - } else { - return "long"; - } - } - - private String getColumnVectorType(String primitiveType) { - if(primitiveType!=null && primitiveType.equals("double")) { - return "DoubleColumnVector"; - } - return "LongColumnVector"; - } - - private String getOutputWritableType(String primitiveType) throws Exception { - if (primitiveType.equals("long")) { - return "LongWritable"; - } else if (primitiveType.equals("double")) { - return "DoubleWritable"; - } - throw new Exception("Unimplemented primitive output writable: " + primitiveType); - } - - private String getOutputObjectInspector(String primitiveType) throws Exception { - if (primitiveType.equals("long")) { - return "PrimitiveObjectInspectorFactory.writableLongObjectInspector"; - } else if (primitiveType.equals("double")) { - return "PrimitiveObjectInspectorFactory.writableDoubleObjectInspector"; - } - throw new Exception("Unimplemented primitive output inspector: " + primitiveType); - } -} - diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ColumnArithmeticColumn.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ColumnArithmeticColumn.txt deleted file mode 100644 index 0e56721..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ColumnArithmeticColumn.txt +++ /dev/null @@ -1,178 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; - -public class extends VectorExpression { - int colNum1; - int colNum2; - int outputColumn; - - public (int colNum1, int colNum2, int outputColumn) { - this.colNum1 = colNum1; - this.colNum2 = colNum2; - this.outputColumn = outputColumn; - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - inputColVector1 = () batch.cols[colNum1]; - inputColVector2 = () batch.cols[colNum2]; - outputColVector = () batch.cols[outputColumn]; - int[] sel = batch.selected; - int n = batch.size; - [] vector1 = inputColVector1.vector; - [] vector2 = inputColVector2.vector; - - [] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - /* Set repeating property to false (the default). - * It will be set to true later if needed later. - */ - outputColVector.isRepeating = false; - - //Handle nulls first - if (inputColVector1.noNulls && !inputColVector2.noNulls) { - outputColVector.noNulls = false; - if (inputColVector2.isRepeating) { - //Output will also be repeating and null - outputColVector.isNull[0] = true; - outputColVector.isRepeating = true; - //return as no further processing is needed - return; - } else { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.isNull[i] = inputColVector2.isNull[i]; - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.isNull[i] = inputColVector2.isNull[i]; - } - } - } - } else if (!inputColVector1.noNulls && inputColVector2.noNulls) { - outputColVector.noNulls = false; - if (inputColVector1.isRepeating) { - //Output will also be repeating and null - outputColVector.isRepeating = true; - outputColVector.isNull[0] = true; - //return as no further processing is needed - return; - } else { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.isNull[i] = inputColVector1.isNull[i]; - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.isNull[i] = inputColVector1.isNull[i]; - } - } - } - } else if (!inputColVector1.noNulls && !inputColVector2.noNulls) { - outputColVector.noNulls = false; - if (inputColVector1.isRepeating || inputColVector2.isRepeating) { - //Output will also be repeating and null - outputColVector.isRepeating = true; - outputColVector.isNull[0] = true; - //return as no further processing is needed - return; - } else { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.isNull[i] = inputColVector1.isNull[i] || inputColVector2.isNull[i]; - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.isNull[i] = inputColVector1.isNull[i] || inputColVector2.isNull[i]; - } - } - } - } - - - //Disregard nulls for processing - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - //All must be selected otherwise size would be zero - //Repeating property will not change. - outputVector[0] = vector1[0] vector2[0]; - outputColVector.isRepeating = true; - } else if (inputColVector1.isRepeating) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1[0] vector2[i]; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = vector1[0] vector2[i]; - } - } - } else if (inputColVector2.isRepeating) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1[i] vector2[0]; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = vector1[i] vector2[0]; - } - } - } else { - if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector1[i] vector2[i]; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = vector1[i] vector2[i]; - } - } - } - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return ""; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ColumnArithmeticScalar.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ColumnArithmeticScalar.txt deleted file mode 100644 index 474a979..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ColumnArithmeticScalar.txt +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.; -import org.apache.hadoop.hive.ql.exec.vector.; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; - -public class extends VectorExpression { - private int colNum; - private value; - private int outputColumn; - - public (int colNum, value, int outputColumn) { - this.colNum = colNum; - this.value = value; - this.outputColumn = outputColumn; - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - inputColVector = () batch.cols[colNum]; - outputColVector = () batch.cols[outputColumn]; - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; - boolean[] outputIsNull = outputColVector.isNull; - outputColVector.noNulls = inputColVector.noNulls; - int n = batch.size; - [] vector = inputColVector.vector; - [] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - if (inputColVector.isRepeating) { - //All must be selected otherwise size would be zero - //Repeating property will not change. - outputVector[0] = vector[0] value; - // Even if there are no nulls, we always copy over entry 0. Simplifies code. - outputIsNull[0] = inputIsNull[0]; - outputColVector.isRepeating = true; - } else if (inputColVector.noNulls) { - if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector[i] value; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = vector[i] value; - } - } - outputColVector.isRepeating = false; - } else /* there are nulls */ { - if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector[i] value; - outputIsNull[i] = inputIsNull[i]; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = vector[i] value; - } - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - } - outputColVector.isRepeating = false; - } - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return ""; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ColumnCompareScalar.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ColumnCompareScalar.txt deleted file mode 100644 index dcf87b0..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ColumnCompareScalar.txt +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.; -import org.apache.hadoop.hive.ql.exec.vector.; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; - -public class extends VectorExpression { - private int colNum; - private value; - private int outputColumn; - - public (int colNum, value, int outputColumn) { - this.colNum = colNum; - this.value = value; - this.outputColumn = outputColumn; - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - inputColVector = () batch.cols[colNum]; - outputColVector = () batch.cols[outputColumn]; - int[] sel = batch.selected; - boolean[] nullPos = inputColVector.isNull; - int n = batch.size; - [] vector = inputColVector.vector; - [] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - if (inputColVector.noNulls) { - if (inputColVector.isRepeating) { - //All must be selected otherwise size would be zero - //Repeating property will not change. - outputVector[0] = vector[0] value ? 1 : 0; - outputColVector.isRepeating = true; - } else if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputVector[i] = vector[i] value ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = vector[i] value ? 1 : 0; - } - } - } else { - if (inputColVector.isRepeating) { - //All must be selected otherwise size would be zero - //Repeating property will not change. - if (!nullPos[0]) { - outputVector[0] = vector[0] value ? 1 : 0; - } else { - outputVector[0] = 0; - } - outputColVector.isRepeating = true; - } else if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - if (!nullPos[i]) { - outputVector[i] = vector[i] value ? 1 : 0; - } else { - //compare with null is false - outputVector[i] = 0; - } - } - } else { - for(int i = 0; i != n; i++) { - if (!nullPos[i]) { - outputVector[i] = vector[i] value ? 1 : 0; - } else { - outputVector[i] = 0; - } - } - } - } - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return ""; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ColumnUnaryMinus.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ColumnUnaryMinus.txt deleted file mode 100644 index 9d16bdd..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ColumnUnaryMinus.txt +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; - -public class extends VectorExpression { - int colNum; - int outputColumn; - - public (int colNum, int outputColumn) { - this.colNum = colNum; - this.outputColumn = outputColumn; - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - this.evaluateChildren(batch); - } - - inputColVector = () batch.cols[colNum]; - outputColVector = () batch.cols[outputColumn]; - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; - boolean[] outputIsNull = outputColVector.isNull; - outputColVector.noNulls = inputColVector.noNulls; - int n = batch.size; - [] vector = inputColVector.vector; - [] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - if (inputColVector.isRepeating) { - //All must be selected otherwise size would be zero - //Repeating property will not change. - outputVector[0] = - vector[0]; - // Even if there are no nulls, we always copy over entry 0. Simplifies code. - outputIsNull[0] = inputIsNull[0]; - outputColVector.isRepeating = true; - } else if (inputColVector.noNulls) { - if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputVector[i] = - vector[i]; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = - vector[i]; - } - } - outputColVector.isRepeating = false; - } else /* there are nulls */ { - if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputVector[i] = - vector[i]; - outputIsNull[i] = inputIsNull[i]; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = - vector[i]; - } - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - } - outputColVector.isRepeating = false; - } - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return ""; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterColumnCompareColumn.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterColumnCompareColumn.txt deleted file mode 100644 index 8e3b717..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterColumnCompareColumn.txt +++ /dev/null @@ -1,227 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; - -public class extends VectorExpression { - int colNum1; - int colNum2; - - public (int colNum1, int colNum2) { - this.colNum1 = colNum1; - this.colNum2 = colNum2; - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - inputColVector1 = () batch.cols[colNum1]; - inputColVector2 = () batch.cols[colNum2]; - int[] sel = batch.selected; - boolean[] nullPos1 = inputColVector1.isNull; - boolean[] nullPos2 = inputColVector2.isNull; - int n = batch.size; - [] vector1 = inputColVector1.vector; - [] vector2 = inputColVector2.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - if (inputColVector1.noNulls && inputColVector2.noNulls) { - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - //All must be selected otherwise size would be zero - //Repeating property will not change. - if (!(vector1[0] vector2[0])) { - batch.size = 0; - } - } else if (inputColVector1.isRepeating) { - if (batch.selectedInUse) { - int newSize = 0; - for(int j=0; j != n; j++) { - int i = sel[j]; - if (vector1[0] vector2[i]) { - sel[newSize++] = i; - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (vector1[0] vector2[i]) { - sel[newSize++] = i; - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else if (inputColVector2.isRepeating) { - if (batch.selectedInUse) { - int newSize = 0; - for(int j=0; j != n; j++) { - int i = sel[j]; - if (vector1[i] vector2[0]) { - sel[newSize++] = i; - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (vector1[i] vector2[0]) { - sel[newSize++] = i; - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else if (batch.selectedInUse) { - int newSize = 0; - for(int j=0; j != n; j++) { - int i = sel[j]; - if (vector1[i] vector2[i]) { - sel[newSize++] = i; - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (vector1[i] vector2[i]) { - sel[newSize++] = i; - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - if (nullPos1[0] || nullPos2[0]) { - batch.size = 0; - } - } else if (inputColVector1.isRepeating) { - if (nullPos1[0]) { - batch.size = 0; - } else { - if (batch.selectedInUse) { - int newSize = 0; - for(int j=0; j != n; j++) { - int i = sel[j]; - if (!nullPos2[i]) { - if (vector1[0] vector2[i]) { - sel[newSize++] = i; - } - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos2[i]) { - if (vector1[0] vector2[i]) { - sel[newSize++] = i; - } - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } - } else if (inputColVector2.isRepeating) { - if (nullPos2[0]) { - batch.size = 0; - } else { - if (batch.selectedInUse) { - int newSize = 0; - for(int j=0; j != n; j++) { - int i = sel[j]; - if (!nullPos1[i]) { - if (vector1[i] vector2[0]) { - sel[newSize++] = i; - } - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos1[i]) { - if (vector1[i] vector2[0]) { - sel[newSize++] = i; - } - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } - } else if (batch.selectedInUse) { - int newSize = 0; - for(int j=0; j != n; j++) { - int i = sel[j]; - if (!nullPos1[i] && !nullPos2[i]) { - if (vector1[i] vector2[i]) { - sel[newSize++] = i; - } - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos1[i] && !nullPos2[i]) { - if (vector1[i] vector2[i]) { - sel[newSize++] = i; - } - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } - - @Override - public String getOutputType() { - return "boolean"; - } - - @Override - public int getOutputColumn() { - return -1; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterColumnCompareScalar.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterColumnCompareScalar.txt deleted file mode 100644 index 04bc69c..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterColumnCompareScalar.txt +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; - - -public class extends VectorExpression { - private int colNum; - private value; - - public (int colNum, value) { - this.colNum = colNum; - this.value = value; - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - inputColVector = () batch.cols[colNum]; - int[] sel = batch.selected; - boolean[] nullPos = inputColVector.isNull; - int n = batch.size; - [] vector = inputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - if (inputColVector.noNulls) { - if (inputColVector.isRepeating) { - //All must be selected otherwise size would be zero - //Repeating property will not change. - if (!(vector[0] value)) { - //Entire batch is filtered out. - batch.size = 0; - } - } else if (batch.selectedInUse) { - int newSize = 0; - for(int j=0; j != n; j++) { - int i = sel[j]; - if (vector[i] value) { - sel[newSize++] = i; - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (vector[i] value) { - sel[newSize++] = i; - } - } - if (newSize < n) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else { - if (inputColVector.isRepeating) { - //All must be selected otherwise size would be zero - //Repeating property will not change. - if (!nullPos[0]) { - if (!(vector[0] value)) { - //Entire batch is filtered out. - batch.size = 0; - } - } else { - batch.size = 0; - } - } else if (batch.selectedInUse) { - int newSize = 0; - for(int j=0; j != n; j++) { - int i = sel[j]; - if (!nullPos[i]) { - if (vector[i] value) { - sel[newSize++] = i; - } - } - } - //Change the selected vector - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos[i]) { - if (vector[i] value) { - sel[newSize++] = i; - } - } - } - if (newSize < n) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } - } - - @Override - public int getOutputColumn() { - return -1; - } - - @Override - public String getOutputType() { - return "boolean"; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterStringColumnCompareColumn.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterStringColumnCompareColumn.txt deleted file mode 100644 index 9309326..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterStringColumnCompareColumn.txt +++ /dev/null @@ -1,455 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; -import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; - -/** - * Filter the rows in a batch by comparing one string column to another. - * This code is generated from a template. - */ -public class extends VectorExpression { - private int colNum1; - private int colNum2; - - public (int colNum1, int colNum2) { - this.colNum1 = colNum1; - this.colNum2 = colNum2; - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - BytesColumnVector inputColVector1 = (BytesColumnVector) batch.cols[colNum1]; - BytesColumnVector inputColVector2 = (BytesColumnVector) batch.cols[colNum2]; - int[] sel = batch.selected; - boolean[] nullPos1 = inputColVector1.isNull; - boolean[] nullPos2 = inputColVector2.isNull; - int n = batch.size; - byte[][] vector1 = inputColVector1.vector; - byte[][] vector2 = inputColVector2.vector; - int[] start1 = inputColVector1.start; - int[] start2 = inputColVector2.start; - int[] length1 = inputColVector1.length; - int[] length2 = inputColVector2.length; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - // handle case where neither input has nulls - if (inputColVector1.noNulls && inputColVector2.noNulls) { - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - - /* Either all must remain selected or all will be eliminated. - * Repeating property will not change. - */ - if (!(StringExpr.compare(vector1[0], start1[0], length1[0], - vector2[0], start2[0], length2[0]) 0)) { - batch.size = 0; - } - } else if (inputColVector1.isRepeating) { - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (StringExpr.compare(vector1[0], start1[0], length1[0], - vector2[i], start2[i], length2[i]) 0) { - sel[newSize++] = i; - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (StringExpr.compare(vector1[0], start1[0], length1[0], - vector2[i], start2[i], length2[i]) 0) { - sel[newSize++] = i; - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else if (inputColVector2.isRepeating) { - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (StringExpr.compare(vector1[i], start1[i], length1[i], - vector2[0], start2[0], length2[0]) 0) { - sel[newSize++] = i; - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (StringExpr.compare(vector1[i], start1[i], length1[i], - vector2[0], start2[0], length2[0]) 0) { - sel[newSize++] = i; - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (StringExpr.compare(vector1[i], start1[i], length1[i], - vector2[i], start2[i], length2[i]) 0) { - sel[newSize++] = i; - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (StringExpr.compare(vector1[i], start1[i], length1[i], - vector2[i], start2[i], length2[i]) 0) { - sel[newSize++] = i; - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - - // handle case where only input 2 has nulls - } else if (inputColVector1.noNulls) { - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - if (nullPos2[0] || - !(StringExpr.compare(vector1[0], start1[0], length1[0], - vector2[0], start2[0], length2[0]) 0)) { - batch.size = 0; - } - } else if (inputColVector1.isRepeating) { - - // no need to check for nulls in input 1 - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (!nullPos2[i]) { - if (StringExpr.compare(vector1[0], start1[0], length1[0], - vector2[i], start2[i], length2[i]) 0) { - sel[newSize++] = i; - } - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos2[i]) { - if (StringExpr.compare(vector1[0], start1[0], length1[0], - vector2[i], start2[i], length2[i]) 0) { - sel[newSize++] = i; - } - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else if (inputColVector2.isRepeating) { - if (nullPos2[0]) { - - // no values will qualify because every comparison will be with NULL - batch.size = 0; - return; - } - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (StringExpr.compare(vector1[i], start1[i], length1[i], - vector2[0], start2[0], length2[0]) 0) { - sel[newSize++] = i; - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (StringExpr.compare(vector1[i], start1[i], length1[i], - vector2[0], start2[0], length2[0]) 0) { - sel[newSize++] = i; - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else { // neither input is repeating - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (!nullPos2[i]) { - if (StringExpr.compare(vector1[i], start1[i], length1[i], - vector2[i], start2[i], length2[i]) 0) { - sel[newSize++] = i; - } - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos2[i]) { - if (StringExpr.compare(vector1[i], start1[i], length1[i], - vector2[i], start2[i], length2[i]) 0) { - sel[newSize++] = i; - } - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } - - // handle case where only input 1 has nulls - } else if (inputColVector2.noNulls) { - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - if (nullPos1[0] || - !(StringExpr.compare(vector1[0], start1[0], length1[0], - vector2[0], start2[0], length2[0]) 0)) { - batch.size = 0; - return; - } - } else if (inputColVector1.isRepeating) { - if (nullPos1[0]) { - - // if repeating value is null then every comparison will fail so nothing qualifies - batch.size = 0; - return; - } - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (StringExpr.compare(vector1[0], start1[0], length1[0], - vector2[i], start2[i], length2[i]) 0) { - sel[newSize++] = i; - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (StringExpr.compare(vector1[0], start1[0], length1[0], - vector2[i], start2[i], length2[i]) 0) { - sel[newSize++] = i; - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else if (inputColVector2.isRepeating) { - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (!nullPos1[i]) { - if (StringExpr.compare(vector1[i], start1[i], length1[i], - vector2[0], start2[0], length2[0]) 0) { - sel[newSize++] = i; - } - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos2[i]) { - if (StringExpr.compare(vector1[i], start1[i], length1[i], - vector2[0], start2[0], length2[0]) 0) { - sel[newSize++] = i; - } - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else { // neither input is repeating - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (!nullPos1[i]) { - if (StringExpr.compare(vector1[i], start1[i], length1[i], - vector2[i], start2[i], length2[i]) 0) { - sel[newSize++] = i; - } - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos1[i]) { - if (StringExpr.compare(vector1[i], start1[i], length1[i], - vector2[i], start2[i], length2[i]) 0) { - sel[newSize++] = i; - } - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } - - // handle case where both inputs have nulls - } else { - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - if (nullPos1[0] || nullPos2[0] || - !(StringExpr.compare(vector1[0], start1[0], length1[0], - vector2[0], start2[0], length2[0]) 0)) { - batch.size = 0; - } - } else if (inputColVector1.isRepeating) { - if (nullPos1[0]) { - batch.size = 0; - return; - } - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (!nullPos2[i]) { - if (StringExpr.compare(vector1[0], start1[0], length1[0], - vector2[i], start2[i], length2[i]) 0) { - sel[newSize++] = i; - } - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos2[i]) { - if (StringExpr.compare(vector1[0], start1[0], length1[0], - vector2[i], start2[i], length2[i]) 0) { - sel[newSize++] = i; - } - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else if (inputColVector2.isRepeating) { - if (nullPos2[0]) { - batch.size = 0; - return; - } - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (!nullPos1[i]) { - if (StringExpr.compare(vector1[i], start1[i], length1[i], - vector2[0], start2[0], length2[0]) 0) { - sel[newSize++] = i; - } - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos1[i]) { - if (StringExpr.compare(vector1[i], start1[i], length1[i], - vector2[0], start2[0], length2[0]) 0) { - sel[newSize++] = i; - } - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else { // neither input is repeating - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (!nullPos1[i] && !nullPos2[i]) { - if (StringExpr.compare(vector1[i], start1[i], length1[i], - vector2[i], start2[i], length2[i]) 0) { - sel[newSize++] = i; - } - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos1[i] && !nullPos2[i]) { - if (StringExpr.compare(vector1[i], start1[i], length1[i], - vector2[i], start2[i], length2[i]) 0) { - sel[newSize++] = i; - } - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } - } - } - - @Override - public String getOutputType() { - return "boolean"; - } - - @Override - public int getOutputColumn() { - return -1; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterStringColumnCompareScalar.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterStringColumnCompareScalar.txt deleted file mode 100644 index 30eddcf..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterStringColumnCompareScalar.txt +++ /dev/null @@ -1,139 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; -import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; - -/** - * This is a generated class to evaluate a comparison on a vector of strings. - */ -public class extends VectorExpression { - private int colNum; - private byte[] value; - - public (int colNum, byte[] value) { - this.colNum = colNum; - this.value = value; - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - if (childExpressions != null) { - super.evaluateChildren(batch); - } - BytesColumnVector inputColVector = (BytesColumnVector) batch.cols[colNum]; - int[] sel = batch.selected; - boolean[] nullPos = inputColVector.isNull; - int n = batch.size; - byte[][] vector = inputColVector.vector; - int[] length = inputColVector.length; - int[] start = inputColVector.start; - - - // return immediately if batch is empty - if (n == 0) { - return; - } - - if (inputColVector.noNulls) { - if (inputColVector.isRepeating) { - - // All must be selected otherwise size would be zero. Repeating property will not change. - if (!(StringExpr.compare(vector[0], start[0], length[0], value, 0, value.length) 0)) { - - //Entire batch is filtered out. - batch.size = 0; - } - } else if (batch.selectedInUse) { - int newSize = 0; - for(int j=0; j != n; j++) { - int i = sel[j]; - if (StringExpr.compare(vector[i], start[i], length[i], value, 0, value.length) 0) { - sel[newSize++] = i; - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (StringExpr.compare(vector[i], start[i], length[i], value, 0, value.length) 0) { - sel[newSize++] = i; - } - } - if (newSize < n) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else { - if (inputColVector.isRepeating) { - - // All must be selected otherwise size would be zero. Repeating property will not change. - if (!nullPos[0]) { - if (!(StringExpr.compare(vector[0], start[0], length[0], value, 0, value.length) 0)) { - - //Entire batch is filtered out. - batch.size = 0; - } - } else { - batch.size = 0; - } - } else if (batch.selectedInUse) { - int newSize = 0; - for(int j=0; j != n; j++) { - int i = sel[j]; - if (!nullPos[i]) { - if (StringExpr.compare(vector[i], start[i], length[i], value, 0, value.length) 0) { - sel[newSize++] = i; - } - } - } - - //Change the selected vector - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos[i]) { - if (StringExpr.compare(vector[i], start[i], length[i], value, 0, value.length) 0) { - sel[newSize++] = i; - } - } - } - if (newSize < n) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } - } - - @Override - public int getOutputColumn() { - return -1; - } - - @Override - public String getOutputType() { - return "boolean"; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ScalarArithmeticColumn.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ScalarArithmeticColumn.txt deleted file mode 100644 index 9899e7e..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/ScalarArithmeticColumn.txt +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; - -/* - * Because of the templatized nature of the code, either or both - * of these ColumnVector imports may be needed. Listing both of them - * rather than using ....vectorization.*; - */ -import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; - -/** - * Implements a vectorized arithmetic operator with a scalar on the left and a - * column vector on the right. The result is output to an output column vector. - */ -public class extends VectorExpression { - private int colNum; - private value; - private int outputColumn; - - public ( value, int colNum, int outputColumn) { - this.colNum = colNum; - this.value = value; - this.outputColumn = outputColumn; - } - - @Override - /** - * Method to evaluate scalar-column operation in vectorized fashion. - * - * @batch a package of rows with each column stored in a vector - */ - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - inputColVector = () batch.cols[colNum]; - outputColVector = () batch.cols[outputColumn]; - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; - boolean[] outputIsNull = outputColVector.isNull; - outputColVector.noNulls = inputColVector.noNulls; - int n = batch.size; - [] vector = inputColVector.vector; - [] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - if (inputColVector.isRepeating) { - - /* - * All must be selected otherwise size would be zero - * Repeating property will not change. - */ - outputVector[0] = value vector[0]; - - // Even if there are no nulls, we always copy over entry 0. Simplifies code. - outputIsNull[0] = inputIsNull[0]; - outputColVector.isRepeating = true; - } else if (inputColVector.noNulls) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = value vector[i]; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = value vector[i]; - } - } - outputColVector.isRepeating = false; - } else { /* there are nulls */ - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = value vector[i]; - outputIsNull[i] = inputIsNull[i]; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = value vector[i]; - } - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - } - outputColVector.isRepeating = false; - } - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return ""; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt deleted file mode 100644 index c8de5de..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import static org.junit.Assert.assertEquals; -import java.util.Random; -import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.util.VectorizedRowGroupGenUtil; -import org.junit.Test; - - -/** - * - * . - * - */ -public class { - - private static final int BATCH_SIZE = 100; - private static final long SEED = 0xfa57; - - - -} - - diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java deleted file mode 100644 index 34c093c..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java +++ /dev/null @@ -1,256 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.templates; - -import java.io.IOException; -import java.util.HashMap; - -/** - * - * TestCodeGen. - * This class is mutable and maintains a hashmap of TestSuiteClassName to test cases. - * The tests cases are added over the course of vectorized expressions class generation, - * with test classes being outputted at the end. For each column vector (inputs and/or outputs) - * a matrix of pairwise covering Booleans is used to generate test cases across nulls and - * repeating dimensions. Based on the input column vector(s) nulls and repeating states - * the states of the output column vector (if there is one) is validated, along with the null - * vector. For filter operations the selection vector is validated against the generated - * data. Each template corresponds to a class representing a test suite. - */ -public class TestCodeGen { - - public enum TestSuiteClassName{ - TestColumnScalarOperationVectorExpressionEvaluation, - TestColumnScalarFilterVectorExpressionEvaluation, - TestColumnColumnOperationVectorExpressionEvaluation, - TestColumnColumnFilterVectorExpressionEvaluation, - } - - private final String testOutputDir; - private final String templateDirectory; - private final HashMap testsuites; - - public TestCodeGen(String testOutputDir, String templateDirectory) { - this.testOutputDir = testOutputDir; - this.templateDirectory = templateDirectory; - testsuites = new HashMap(); - - for(TestSuiteClassName className : TestSuiteClassName.values()) { - testsuites.put(className,new StringBuilder()); - } - - } - - public void addColumnScalarOperationTestCases(boolean op1IsCol, String vectorExpClassName, - String inputColumnVectorType, String outputColumnVectorType, String scalarType) - throws IOException { - - TestSuiteClassName template = - TestSuiteClassName.TestColumnScalarOperationVectorExpressionEvaluation; - - //Read the template into a string; - String templateFile = CodeGen.joinPath(this.templateDirectory,template.toString()+".txt"); - String templateString = removeTemplateComments(CodeGen.readFile(templateFile)); - - for(Boolean[] testMatrix :new Boolean[][]{ - // Pairwise: InitOuputColHasNulls, InitOuputColIsRepeating, ColumnHasNulls, ColumnIsRepeating - {false, true, true, true}, - {false, false, false, false}, - {true, false, true, false}, - {true, true, false, false}, - {true, false, false, true}}) { - String testCase = templateString; - testCase = testCase.replaceAll("", - "test" - + vectorExpClassName - + createNullRepeatingNameFragment("Out", testMatrix[0], testMatrix[1]) - + createNullRepeatingNameFragment("Col", testMatrix[2], testMatrix[3])); - testCase = testCase.replaceAll("", vectorExpClassName); - testCase = testCase.replaceAll("", inputColumnVectorType); - testCase = testCase.replaceAll("", outputColumnVectorType); - testCase = testCase.replaceAll("", scalarType); - testCase = testCase.replaceAll("", CodeGen.getCamelCaseType(scalarType)); - testCase = testCase.replaceAll("", testMatrix[0].toString()); - testCase = testCase.replaceAll("", testMatrix[1].toString()); - testCase = testCase.replaceAll("", testMatrix[2].toString()); - testCase = testCase.replaceAll("", testMatrix[3].toString()); - - if(op1IsCol){ - testCase = testCase.replaceAll("","0, scalarValue"); - }else{ - testCase = testCase.replaceAll("","scalarValue, 0"); - } - - testsuites.get(template).append(testCase); - } - } - - public void addColumnScalarFilterTestCases(boolean op1IsCol, String vectorExpClassName, - String inputColumnVectorType, String scalarType, String operatorSymbol) - throws IOException { - - TestSuiteClassName template = - TestSuiteClassName.TestColumnScalarFilterVectorExpressionEvaluation; - - //Read the template into a string; - String templateFile = CodeGen.joinPath(this.templateDirectory,template.toString()+".txt"); - String templateString = removeTemplateComments(CodeGen.readFile(templateFile)); - - for(Boolean[] testMatrix : new Boolean[][]{ - // Pairwise: ColumnHasNulls, ColumnIsRepeating - {true, true}, - {true, false}, - {false, false}, - {false, true}}) { - String testCase = templateString; - testCase = testCase.replaceAll("", - "test" - + vectorExpClassName - + createNullRepeatingNameFragment("Col", testMatrix[0], testMatrix[1])); - testCase = testCase.replaceAll("", vectorExpClassName); - testCase = testCase.replaceAll("", inputColumnVectorType); - testCase = testCase.replaceAll("", scalarType); - testCase = testCase.replaceAll("", CodeGen.getCamelCaseType(scalarType)); - testCase = testCase.replaceAll("", testMatrix[0].toString()); - testCase = testCase.replaceAll("", testMatrix[1].toString()); - testCase = testCase.replaceAll("", operatorSymbol); - - if(op1IsCol){ - testCase = testCase.replaceAll("","0, scalarValue"); - }else{ - testCase = testCase.replaceAll("","scalarValue, 0"); - } - - testsuites.get(template).append(testCase); - } - } - - public void addColumnColumnOperationTestCases(String vectorExpClassName, - String inputColumnVectorType1, String inputColumnVectorType2, String outputColumnVectorType) - throws IOException { - - TestSuiteClassName template= - TestSuiteClassName.TestColumnColumnOperationVectorExpressionEvaluation; - - //Read the template into a string; - String templateFile = CodeGen.joinPath(this.templateDirectory,template.toString()+".txt"); - String templateString = removeTemplateComments(CodeGen.readFile(templateFile)); - - for(Boolean[] testMatrix : new Boolean[][]{ - // Pairwise: InitOuputColHasNulls, InitOuputColIsRepeating, Column1HasNulls, - // Column1IsRepeating, Column2HasNulls, Column2IsRepeating - {true, true, false, true, true, true}, - {false, false, true, false, false, false}, - {true, false, true, false, true, true}, - {true, true, true, true, false, false}, - {false, false, false, true, true, false}, - {false, true, false, false, false, true}}) { - String testCase = templateString; - testCase = testCase.replaceAll("", - "test" - + vectorExpClassName - + createNullRepeatingNameFragment("Out", testMatrix[0], testMatrix[1]) - + createNullRepeatingNameFragment("C1", testMatrix[2], testMatrix[3]) - + createNullRepeatingNameFragment("C2", testMatrix[4], testMatrix[5])); - testCase = testCase.replaceAll("", vectorExpClassName); - testCase = testCase.replaceAll("", inputColumnVectorType1); - testCase = testCase.replaceAll("", inputColumnVectorType2); - testCase = testCase.replaceAll("", outputColumnVectorType); - testCase = testCase.replaceAll("", testMatrix[0].toString()); - testCase = testCase.replaceAll("", testMatrix[1].toString()); - testCase = testCase.replaceAll("", testMatrix[2].toString()); - testCase = testCase.replaceAll("", testMatrix[3].toString()); - testCase = testCase.replaceAll("", testMatrix[4].toString()); - testCase = testCase.replaceAll("", testMatrix[5].toString()); - - testsuites.get(template).append(testCase); - } - } - - public void addColumnColumnFilterTestCases(String vectorExpClassName, - String inputColumnVectorType1, String inputColumnVectorType2, String operatorSymbol) - throws IOException { - - TestSuiteClassName template= - TestSuiteClassName.TestColumnColumnFilterVectorExpressionEvaluation; - - //Read the template into a string; - String templateFile = CodeGen.joinPath(this.templateDirectory,template.toString()+".txt"); - String templateString = removeTemplateComments(CodeGen.readFile(templateFile)); - - for(Boolean[] testMatrix : new Boolean[][]{ - // Pairwise: Column1HasNulls, Column1IsRepeating, Column2HasNulls, Column2IsRepeating - {false, true, true, true}, - {false, false, false, false}, - {true, false, true, false}, - {true, true, false, false}, - {true, false, false, true}}) { - String testCase = templateString; - testCase = testCase.replaceAll("", - "test" - + vectorExpClassName - + createNullRepeatingNameFragment("C1", testMatrix[0], testMatrix[1]) - + createNullRepeatingNameFragment("C2", testMatrix[2], testMatrix[3])); - testCase = testCase.replaceAll("", vectorExpClassName); - testCase = testCase.replaceAll("", inputColumnVectorType1); - testCase = testCase.replaceAll("", inputColumnVectorType2); - testCase = testCase.replaceAll("", testMatrix[0].toString()); - testCase = testCase.replaceAll("", testMatrix[1].toString()); - testCase = testCase.replaceAll("", testMatrix[2].toString()); - testCase = testCase.replaceAll("", testMatrix[3].toString()); - testCase = testCase.replaceAll("", operatorSymbol); - - testsuites.get(template).append(testCase); - } - } - - public void generateTestSuites() throws IOException { - - String templateFile = CodeGen.joinPath(this.templateDirectory, "TestClass.txt"); - for(TestSuiteClassName testClass : testsuites.keySet()) { - - String templateString = CodeGen.readFile(templateFile); - templateString = templateString.replaceAll("", testClass.toString()); - templateString = templateString.replaceAll("", testsuites.get(testClass).toString()); - - String outputFile = CodeGen.joinPath(this.testOutputDir, testClass + ".java"); - - CodeGen.writeFile(outputFile, templateString); - } - } - - private static String createNullRepeatingNameFragment(String idenitfier, boolean nulls, boolean repeating) - { - if(nulls || repeating){ - if(nulls){ - idenitfier+="Nulls"; - } - if(repeating){ - idenitfier+="Repeats"; - } - return idenitfier; - } - - return ""; - } - - private static String removeTemplateComments(String templateString){ - return templateString.replaceAll("(?s)", ""); - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt deleted file mode 100644 index 2bb1aa3..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt +++ /dev/null @@ -1,69 +0,0 @@ - - @Test - public void () { - - Random rand = new Random(SEED); - - inputColumnVector1 = - VectorizedRowGroupGenUtil.generate(, - , BATCH_SIZE, rand); - - inputColumnVector2 = - VectorizedRowGroupGenUtil.generate(, - , BATCH_SIZE, rand); - - VectorizedRowBatch rowBatch = new VectorizedRowBatch(2, BATCH_SIZE); - rowBatch.cols[0] = inputColumnVector1; - rowBatch.cols[1] = inputColumnVector2; - - vectorExpression = - new (0, 1); - - vectorExpression.evaluate(rowBatch); - - int selectedIndex = 0; - for(int i = 0; i < BATCH_SIZE; i++) { - //null vector is safe to check, as it is always initialized to match the data vector - if(!inputColumnVector1.isNull[i] && !inputColumnVector2.isNull[i]) { - if(inputColumnVector1.vector[i] inputColumnVector2.vector[i]) { - assertEquals( - "Vector index that passes filter " - + inputColumnVector1.vector[i] + "" - + inputColumnVector2.vector[i] + " is not in rowBatch selected index", - i, - rowBatch.selected[selectedIndex]); - selectedIndex++; - } - } - } - - assertEquals("Row batch size not set to number of selected rows: " + selectedIndex, - selectedIndex, rowBatch.size); - - if(selectedIndex > 0 && selectedIndex < BATCH_SIZE) { - assertEquals( - "selectedInUse should be set when > 0 and < entire batch(" + BATCH_SIZE + ") is selected: " - + selectedIndex, - true, rowBatch.selectedInUse); - } else if(selectedIndex == BATCH_SIZE) { - assertEquals( - "selectedInUse should not be set when entire batch(" + BATCH_SIZE + ") is selected: " - + selectedIndex, - false, rowBatch.selectedInUse); - } - } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt deleted file mode 100644 index 4ab3e76..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt +++ /dev/null @@ -1,64 +0,0 @@ - - @Test - public void () { - - Random rand = new Random(SEED); - - outputColumnVector = - VectorizedRowGroupGenUtil.generate(, - , BATCH_SIZE, rand); - - inputColumnVector1 = - VectorizedRowGroupGenUtil.generate(, - , BATCH_SIZE, rand); - - inputColumnVector2 = - VectorizedRowGroupGenUtil.generate(, - , BATCH_SIZE, rand); - - VectorizedRowBatch rowBatch = new VectorizedRowBatch(3, BATCH_SIZE); - rowBatch.cols[0] = inputColumnVector1; - rowBatch.cols[1] = inputColumnVector2; - rowBatch.cols[2] = outputColumnVector; - - vectorExpression = - new (0, 1, 2); - - vectorExpression.evaluate(rowBatch); - - assertEquals( - "Output column vector repeating state does not match operand columns", - (!inputColumnVector1.noNulls && inputColumnVector1.isRepeating) - || (!inputColumnVector2.noNulls && inputColumnVector2.isRepeating) - || inputColumnVector1.isRepeating && inputColumnVector2.isRepeating, - outputColumnVector.isRepeating); - - assertEquals( - "Output column vector no nulls state does not match operand columns", - inputColumnVector1.noNulls && inputColumnVector2.noNulls, outputColumnVector.noNulls); - - //if repeating, only the first value matters - if(!outputColumnVector.noNulls && !outputColumnVector.isRepeating) { - for(int i = 0; i < BATCH_SIZE; i++) { - //null vectors are safe to check, as they are always initialized to match the data vector - assertEquals("Output vector doesn't match input vectors' is null state for index", - inputColumnVector1.isNull[i] || inputColumnVector2.isNull[i], - outputColumnVector.isNull[i]); - } - } - } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt deleted file mode 100644 index 5b53d6a..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt +++ /dev/null @@ -1,76 +0,0 @@ - - @Test - public void () { - - Random rand = new Random(SEED); - - inputColumnVector = - VectorizedRowGroupGenUtil.generate(, - , BATCH_SIZE, rand); - - VectorizedRowBatch rowBatch = new VectorizedRowBatch(1, BATCH_SIZE); - rowBatch.cols[0] = inputColumnVector; - - scalarValue = 0; - do { - scalarValue = rand.next(); - } while(scalarValue == 0); - - vectorExpression = - new (); - - vectorExpression.evaluate(rowBatch); - - int selectedIndex = 0; - //check for isRepeating optimization - if(inputColumnVector.isRepeating) { - //null vector is safe to check, as it is always initialized to match the data vector - selectedIndex = - !inputColumnVector.isNull[0] && inputColumnVector.vector[0] scalarValue - ? BATCH_SIZE : 0; - } else { - for(int i = 0; i < BATCH_SIZE; i++) { - if(!inputColumnVector.isNull[i]) { - if(inputColumnVector.vector[i] scalarValue) { - assertEquals( - "Vector index that passes filter " - + inputColumnVector.vector[i] + "" - + scalarValue + " is not in rowBatch selected index", - i, - rowBatch.selected[selectedIndex]); - selectedIndex++; - } - } - } - } - - assertEquals("Row batch size not set to number of selected rows: " + selectedIndex, - selectedIndex, rowBatch.size); - - if(selectedIndex > 0 && selectedIndex < BATCH_SIZE) { - assertEquals( - "selectedInUse should be set when > 0 and < entire batch(" + BATCH_SIZE + ") is selected: " - + selectedIndex, - true, rowBatch.selectedInUse); - } else if(selectedIndex == BATCH_SIZE) { - assertEquals( - "selectedInUse should not be set when entire batch(" + BATCH_SIZE + ") is selected: " - + selectedIndex, - false, rowBatch.selectedInUse); - } - } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt deleted file mode 100644 index e5f3f18..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt +++ /dev/null @@ -1,59 +0,0 @@ - - @Test - public void () { - - Random rand = new Random(SEED); - - outputColumnVector = - VectorizedRowGroupGenUtil.generate(, - , BATCH_SIZE, rand); - - inputColumnVector = - VectorizedRowGroupGenUtil.generate(, - , BATCH_SIZE, rand); - - VectorizedRowBatch rowBatch = new VectorizedRowBatch(2, BATCH_SIZE); - rowBatch.cols[0] = inputColumnVector; - rowBatch.cols[1] = outputColumnVector; - - scalarValue = 0; - do { - scalarValue = rand.next(); - } while(scalarValue == 0); - - vectorExpression = - new (, 1); - - vectorExpression.evaluate(rowBatch); - - assertEquals( - "Output column vector is repeating state does not match operand column", - inputColumnVector.isRepeating, outputColumnVector.isRepeating); - - assertEquals( - "Output column vector no nulls state does not match operand column", - inputColumnVector.noNulls, outputColumnVector.noNulls); - - if(!outputColumnVector.noNulls && !outputColumnVector.isRepeating) { - for(int i = 0; i < BATCH_SIZE; i++) { - //null vectors are safe to check, as they are always initialized to match the data vector - assertEquals("Output vector doesn't match input vector's is null state for index", - inputColumnVector.isNull[i], outputColumnVector.isNull[i]); - } - } - } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFAvg.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFAvg.txt deleted file mode 100644 index d85346d..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFAvg.txt +++ /dev/null @@ -1,440 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen; - -import java.util.ArrayList; - -import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates. - VectorAggregateExpression.AggregationBuffer; -import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.hive.serde2.io.DoubleWritable; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; - -import org.apache.hadoop.hive.ql.io.orc.*; - -@Description(name = "avg", value = "_FUNC_(expr) - Returns the average value of expr (vectorized, type: )") -public class extends VectorAggregateExpression { - - /** class for storing the current aggregate value. */ - static class Aggregation implements AggregationBuffer { - sum; - long count; - boolean isNull; - - public void sumValue( value) { - if (isNull) { - sum = value; - count = 1; - isNull = false; - } else { - sum += value; - count++; - } - } - } - - private VectorExpression inputExpression; - private Object[] partialResult; - private LongWritable resultCount; - private DoubleWritable resultSum; - private StructObjectInspector soi; - - public (VectorExpression inputExpression) { - super(); - this.inputExpression = inputExpression; - partialResult = new Object[2]; - resultCount = new LongWritable(); - resultSum = new DoubleWritable(); - partialResult[0] = resultCount; - partialResult[1] = resultSum; - - initPartialResultInspector(); - } - - private void initPartialResultInspector() { - ArrayList foi = new ArrayList(); - foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector); - foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); - ArrayList fname = new ArrayList(); - fname.add("count"); - fname.add("sum"); - soi = ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi); - } - - private Aggregation getCurrentAggregationBuffer( - VectorAggregationBufferRow[] aggregationBufferSets, - int bufferIndex, - int row) { - VectorAggregationBufferRow mySet = aggregationBufferSets[row]; - Aggregation myagg = (Aggregation) mySet.getAggregationBuffer(bufferIndex); - return myagg; - } - - @Override - public void aggregateInputSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int bufferIndex, - VectorizedRowBatch batch) throws HiveException { - - int batchSize = batch.size; - - if (batchSize == 0) { - return; - } - - inputExpression.evaluate(batch); - - LongColumnVector inputVector = (LongColumnVector)batch. - cols[this.inputExpression.getOutputColumn()]; - long[] vector = inputVector.vector; - - if (inputVector.noNulls) { - if (inputVector.isRepeating) { - iterateNoNullsRepeatingWithAggregationSelection( - aggregationBufferSets, bufferIndex, - vector[0], batchSize); - } else { - if (batch.selectedInUse) { - iterateNoNullsSelectionWithAggregationSelection( - aggregationBufferSets, bufferIndex, - vector, batch.selected, batchSize); - } else { - iterateNoNullsWithAggregationSelection( - aggregationBufferSets, bufferIndex, - vector, batchSize); - } - } - } else { - if (inputVector.isRepeating) { - if (batch.selectedInUse) { - iterateHasNullsRepeatingSelectionWithAggregationSelection( - aggregationBufferSets, bufferIndex, - vector[0], batchSize, batch.selected, inputVector.isNull); - } else { - iterateHasNullsRepeatingWithAggregationSelection( - aggregationBufferSets, bufferIndex, - vector[0], batchSize, inputVector.isNull); - } - } else { - if (batch.selectedInUse) { - iterateHasNullsSelectionWithAggregationSelection( - aggregationBufferSets, bufferIndex, - vector, batchSize, batch.selected, inputVector.isNull); - } else { - iterateHasNullsWithAggregationSelection( - aggregationBufferSets, bufferIndex, - vector, batchSize, inputVector.isNull); - } - } - } - } - - private void iterateNoNullsRepeatingWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int bufferIndex, - long value, - int batchSize) { - - for (int i=0; i < batchSize; ++i) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - bufferIndex, - i); - myagg.sumValue(value); - } - } - - private void iterateNoNullsSelectionWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int bufferIndex, - long[] values, - int[] selection, - int batchSize) { - - for (int i=0; i < batchSize; ++i) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - bufferIndex, - i); - myagg.sumValue(values[selection[i]]); - } - } - - private void iterateNoNullsWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int bufferIndex, - long[] values, - int batchSize) { - for (int i=0; i < batchSize; ++i) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - bufferIndex, - i); - myagg.sumValue(values[i]); - } - } - - private void iterateHasNullsRepeatingSelectionWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int bufferIndex, - long value, - int batchSize, - int[] selection, - boolean[] isNull) { - - for (int i=0; i < batchSize; ++i) { - if (!isNull[selection[i]]) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - bufferIndex, - i); - myagg.sumValue(value); - } - } - - } - - private void iterateHasNullsRepeatingWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int bufferIndex, - long value, - int batchSize, - boolean[] isNull) { - - for (int i=0; i < batchSize; ++i) { - if (!isNull[i]) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - bufferIndex, - i); - myagg.sumValue(value); - } - } - } - - private void iterateHasNullsSelectionWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int bufferIndex, - long[] values, - int batchSize, - int[] selection, - boolean[] isNull) { - - for (int j=0; j < batchSize; ++j) { - int i = selection[j]; - if (!isNull[i]) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - bufferIndex, - j); - myagg.sumValue(values[i]); - } - } - } - - private void iterateHasNullsWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int bufferIndex, - long[] values, - int batchSize, - boolean[] isNull) { - - for (int i=0; i < batchSize; ++i) { - if (!isNull[i]) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - bufferIndex, - i); - myagg.sumValue(values[i]); - } - } - } - - - @Override - public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) throws HiveException { - - inputExpression.evaluate(batch); - - inputVector = ()batch.cols[this.inputExpression.getOutputColumn()]; - - int batchSize = batch.size; - - if (batchSize == 0) { - return; - } - - Aggregation myagg = (Aggregation)agg; - - [] vector = inputVector.vector; - - if (inputVector.isRepeating) { - if (inputVector.noNulls || !inputVector.isNull[0]) { - if (myagg.isNull) { - myagg.isNull = false; - myagg.sum = 0; - myagg.count = 0; - } - myagg.sum += vector[0]*batchSize; - myagg.count += batchSize; - } - return; - } - - if (!batch.selectedInUse && inputVector.noNulls) { - iterateNoSelectionNoNulls(myagg, vector, batchSize); - } - else if (!batch.selectedInUse) { - iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull); - } - else if (inputVector.noNulls){ - iterateSelectionNoNulls(myagg, vector, batchSize, batch.selected); - } - else { - iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, batch.selected); - } - } - - private void iterateSelectionHasNulls( - Aggregation myagg, - [] vector, - int batchSize, - boolean[] isNull, - int[] selected) { - - for (int j=0; j< batchSize; ++j) { - int i = selected[j]; - if (!isNull[i]) { - value = vector[i]; - if (myagg.isNull) { - myagg.isNull = false; - myagg.sum = 0; - myagg.count = 0; - } - myagg.sum += value; - myagg.count += 1; - } - } - } - - private void iterateSelectionNoNulls( - Aggregation myagg, - [] vector, - int batchSize, - int[] selected) { - - if (myagg.isNull) { - myagg.isNull = false; - myagg.sum = 0; - myagg.count = 0; - } - - for (int i=0; i< batchSize; ++i) { - value = vector[selected[i]]; - myagg.sum += value; - myagg.count += 1; - } - } - - private void iterateNoSelectionHasNulls( - Aggregation myagg, - [] vector, - int batchSize, - boolean[] isNull) { - - for(int i=0;i value = vector[i]; - if (myagg.isNull) { - myagg.isNull = false; - myagg.sum = 0; - myagg.count = 0; - } - myagg.sum += value; - myagg.count += 1; - } - } - } - - private void iterateNoSelectionNoNulls( - Aggregation myagg, - [] vector, - int batchSize) { - if (myagg.isNull) { - myagg.isNull = false; - myagg.sum = 0; - myagg.count = 0; - } - - for (int i=0;i value = vector[i]; - myagg.sum += value; - myagg.count += 1; - } - } - - @Override - public AggregationBuffer getNewAggregationBuffer() throws HiveException { - return new Aggregation(); - } - - @Override - public void reset(AggregationBuffer agg) throws HiveException { - Aggregation myAgg = (Aggregation) agg; - myAgg.isNull = true; - } - - @Override - public Object evaluateOutput( - AggregationBuffer agg) throws HiveException { - Aggregation myagg = (Aggregation) agg; - if (myagg.isNull) { - return null; - } - else { - assert(0 < myagg.count); - resultCount.set (myagg.count); - resultSum.set (myagg.sum); - return partialResult; - } - } - - @Override - public ObjectInspector getOutputObjectInspector() { - return soi; - } -} - diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFCount.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFCount.txt deleted file mode 100644 index 54c80ec..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFCount.txt +++ /dev/null @@ -1,262 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen; - -import java.util.ArrayList; - -import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates. - VectorAggregateExpression.AggregationBuffer; -import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; - -/** -* . Vectorized implementation for COUNT aggregates. -*/ -@Description(name = "count", value = "_FUNC_(expr) - Returns the maximum value of expr (vectorized, type: )") -public class extends VectorAggregateExpression { - - /** - /* class for storing the current aggregate value. - */ - static class Aggregation implements AggregationBuffer { - long value; - boolean isNull; - - public void initIfNull() { - if (isNull) { - isNull = false; - value = 0; - } - } - } - - private VectorExpression inputExpression; - private LongWritable result; - - public (VectorExpression inputExpression) { - super(); - this.inputExpression = inputExpression; - result = new LongWritable(0); - } - - private Aggregation getCurrentAggregationBuffer( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - int row) { - VectorAggregationBufferRow mySet = aggregationBufferSets[row]; - Aggregation myagg = (Aggregation) mySet.getAggregationBuffer(aggregateIndex); - return myagg; - } - - @Override - public void aggregateInputSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - VectorizedRowBatch batch) throws HiveException { - - int batchSize = batch.size; - - if (batchSize == 0) { - return; - } - - inputExpression.evaluate(batch); - - inputVector = ()batch. - cols[this.inputExpression.getOutputColumn()]; - - if (inputVector.noNulls) { - // if there are no nulls then the iteration is the same on all cases - iterateNoNullsWithAggregationSelection( - aggregationBufferSets, aggregateIndex, batchSize); - } else if (!batch.selectedInUse) { - iterateHasNullsWithAggregationSelection( - aggregationBufferSets, aggregateIndex, - batchSize, inputVector.isNull); - } else if (batch.selectedInUse) { - iterateHasNullsSelectionWithAggregationSelection( - aggregationBufferSets, aggregateIndex, - batchSize, batch.selected, inputVector.isNull); - } - } - - private void iterateNoNullsWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - int batchSize) { - - for (int i=0; i < batchSize; ++i) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregateIndex, - i); - myagg.initIfNull(); - myagg.value++; - } - } - - private void iterateHasNullsWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - int batchSize, - boolean[] isNull) { - - for (int i=0; i < batchSize; ++i) { - if (!isNull[i]) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregateIndex, - i); - myagg.initIfNull(); - myagg.value++; - } - } - } - - private void iterateHasNullsSelectionWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - int batchSize, - int[] selection, - boolean[] isNull) { - - for (int j=0; j < batchSize; ++j) { - int i = selection[j]; - if (!isNull[i]) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregateIndex, - j); - myagg.initIfNull(); - myagg.value++; - } - } - } - - - @Override - public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) - throws HiveException { - - inputExpression.evaluate(batch); - - inputVector = ()batch. - cols[this.inputExpression.getOutputColumn()]; - - int batchSize = batch.size; - - if (batchSize == 0) { - return; - } - - Aggregation myagg = (Aggregation)agg; - - myagg.initIfNull(); - - if (inputVector.isRepeating) { - if (inputVector.noNulls || !inputVector.isNull[0]) { - myagg.value += batchSize; - } - return; - } - - if (inputVector.noNulls) { - myagg.value += batchSize; - return; - } - else if (!batch.selectedInUse) { - iterateNoSelectionHasNulls(myagg, batchSize, inputVector.isNull); - } - else { - iterateSelectionHasNulls(myagg, batchSize, inputVector.isNull, batch.selected); - } - } - - private void iterateSelectionHasNulls( - Aggregation myagg, - int batchSize, - boolean[] isNull, - int[] selected) { - - for (int j=0; j< batchSize; ++j) { - int i = selected[j]; - if (!isNull[i]) { - myagg.value += 1; - } - } - } - - private void iterateNoSelectionHasNulls( - Aggregation myagg, - int batchSize, - boolean[] isNull) { - - for (int i=0; i< batchSize; ++i) { - if (!isNull[i]) { - myagg.value += 1; - } - } - } - - @Override - public AggregationBuffer getNewAggregationBuffer() throws HiveException { - return new Aggregation(); - } - - @Override - public void reset(AggregationBuffer agg) throws HiveException { - Aggregation myAgg = (Aggregation) agg; - myAgg.isNull = true; - } - - @Override - public Object evaluateOutput(AggregationBuffer agg) throws HiveException { - Aggregation myagg = (Aggregation) agg; - if (myagg.isNull) { - return null; - } - else { - result.set (myagg.value); - return result; - } - } - - @Override - public ObjectInspector getOutputObjectInspector() { - return PrimitiveObjectInspectorFactory.writableLongObjectInspector; - } - -} - diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMax.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMax.txt deleted file mode 100644 index d00d9ae..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMax.txt +++ /dev/null @@ -1,415 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen; - -import java.util.ArrayList; - -import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates. - VectorAggregateExpression.AggregationBuffer; -import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.hive.serde2.io.DoubleWritable; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; - -/** -* . Vectorized implementation for MIN/MAX aggregates. -*/ -@Description(name = "", value = "") -public class extends VectorAggregateExpression { - - /** - /* class for storing the current aggregate value. - */ - static private final class Aggregation implements AggregationBuffer { - value; - boolean isNull; - - public void checkValue( value) { - if (isNull) { - isNull = false; - this.value = value; - } else if (value this.value) { - this.value = value; - } - } - } - - private VectorExpression inputExpression; - private result; - - public (VectorExpression inputExpression) { - super(); - this.inputExpression = inputExpression; - result = new (); - } - - private Aggregation getCurrentAggregationBuffer( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregrateIndex, - int row) { - VectorAggregationBufferRow mySet = aggregationBufferSets[row]; - Aggregation myagg = (Aggregation) mySet.getAggregationBuffer(aggregrateIndex); - return myagg; - } - -@Override - public void aggregateInputSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregrateIndex, - VectorizedRowBatch batch) throws HiveException { - - int batchSize = batch.size; - - if (batchSize == 0) { - return; - } - - inputExpression.evaluate(batch); - - inputVector = ()batch. - cols[this.inputExpression.getOutputColumn()]; - [] vector = inputVector.vector; - - if (inputVector.noNulls) { - if (inputVector.isRepeating) { - iterateNoNullsRepeatingWithAggregationSelection( - aggregationBufferSets, aggregrateIndex, - vector[0], batchSize); - } else { - if (batch.selectedInUse) { - iterateNoNullsSelectionWithAggregationSelection( - aggregationBufferSets, aggregrateIndex, - vector, batch.selected, batchSize); - } else { - iterateNoNullsWithAggregationSelection( - aggregationBufferSets, aggregrateIndex, - vector, batchSize); - } - } - } else { - if (inputVector.isRepeating) { - if (batch.selectedInUse) { - iterateHasNullsRepeatingSelectionWithAggregationSelection( - aggregationBufferSets, aggregrateIndex, - vector[0], batchSize, batch.selected, inputVector.isNull); - } else { - iterateHasNullsRepeatingWithAggregationSelection( - aggregationBufferSets, aggregrateIndex, - vector[0], batchSize, inputVector.isNull); - } - } else { - if (batch.selectedInUse) { - iterateHasNullsSelectionWithAggregationSelection( - aggregationBufferSets, aggregrateIndex, - vector, batchSize, batch.selected, inputVector.isNull); - } else { - iterateHasNullsWithAggregationSelection( - aggregationBufferSets, aggregrateIndex, - vector, batchSize, inputVector.isNull); - } - } - } - } - - private void iterateNoNullsRepeatingWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregrateIndex, - value, - int batchSize) { - - for (int i=0; i < batchSize; ++i) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregrateIndex, - i); - myagg.checkValue(value); - } - } - - private void iterateNoNullsSelectionWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregrateIndex, - [] values, - int[] selection, - int batchSize) { - - for (int i=0; i < batchSize; ++i) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregrateIndex, - i); - myagg.checkValue(values[selection[i]]); - } - } - - private void iterateNoNullsWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregrateIndex, - [] values, - int batchSize) { - for (int i=0; i < batchSize; ++i) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregrateIndex, - i); - myagg.checkValue(values[i]); - } - } - - private void iterateHasNullsRepeatingSelectionWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregrateIndex, - value, - int batchSize, - int[] selection, - boolean[] isNull) { - - for (int i=0; i < batchSize; ++i) { - if (!isNull[selection[i]]) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregrateIndex, - i); - myagg.checkValue(value); - } - } - - } - - private void iterateHasNullsRepeatingWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregrateIndex, - value, - int batchSize, - boolean[] isNull) { - - for (int i=0; i < batchSize; ++i) { - if (!isNull[i]) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregrateIndex, - i); - myagg.checkValue(value); - } - } - } - - private void iterateHasNullsSelectionWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregrateIndex, - [] values, - int batchSize, - int[] selection, - boolean[] isNull) { - - for (int j=0; j < batchSize; ++j) { - int i = selection[j]; - if (!isNull[i]) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregrateIndex, - j); - myagg.checkValue(values[i]); - } - } - } - - private void iterateHasNullsWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregrateIndex, - [] values, - int batchSize, - boolean[] isNull) { - - for (int i=0; i < batchSize; ++i) { - if (!isNull[i]) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregrateIndex, - i); - myagg.checkValue(values[i]); - } - } - } - - @Override - public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) - throws HiveException { - - inputExpression.evaluate(batch); - - inputVector = ()batch. - cols[this.inputExpression.getOutputColumn()]; - - int batchSize = batch.size; - - if (batchSize == 0) { - return; - } - - Aggregation myagg = (Aggregation)agg; - - [] vector = inputVector.vector; - - if (inputVector.isRepeating) { - if ((inputVector.noNulls || !inputVector.isNull[0]) && - myagg.isNull || vector[0] < myagg.value) { - myagg.isNull = false; - myagg.value = vector[0]; - } - return; - } - - if (!batch.selectedInUse && inputVector.noNulls) { - iterateNoSelectionNoNulls(myagg, vector, batchSize); - } - else if (!batch.selectedInUse) { - iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull); - } - else if (inputVector.noNulls){ - iterateSelectionNoNulls(myagg, vector, batchSize, batch.selected); - } - else { - iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, batch.selected); - } - } - - private void iterateSelectionHasNulls( - Aggregation myagg, - [] vector, - int batchSize, - boolean[] isNull, - int[] selected) { - - for (int j=0; j< batchSize; ++j) { - int i = selected[j]; - if (!isNull[i]) { - value = vector[i]; - if (myagg.isNull) { - myagg.isNull = false; - myagg.value = value; - } - else if (value myagg.value) { - myagg.value = value; - } - } - } - } - - private void iterateSelectionNoNulls( - Aggregation myagg, - [] vector, - int batchSize, - int[] selected) { - - if (myagg.isNull) { - myagg.value = vector[selected[0]]; - myagg.isNull = false; - } - - for (int i=0; i< batchSize; ++i) { - value = vector[selected[i]]; - if (value myagg.value) { - myagg.value = value; - } - } - } - - private void iterateNoSelectionHasNulls( - Aggregation myagg, - [] vector, - int batchSize, - boolean[] isNull) { - - for(int i=0;i value = vector[i]; - if (myagg.isNull) { - myagg.value = value; - myagg.isNull = false; - } - else if (value myagg.value) { - myagg.value = value; - } - } - } - } - - private void iterateNoSelectionNoNulls( - Aggregation myagg, - [] vector, - int batchSize) { - if (myagg.isNull) { - myagg.value = vector[0]; - myagg.isNull = false; - } - - for (int i=0;i value = vector[i]; - if (value myagg.value) { - myagg.value = value; - } - } - } - - @Override - public AggregationBuffer getNewAggregationBuffer() throws HiveException { - return new Aggregation(); - } - - @Override - public void reset(AggregationBuffer agg) throws HiveException { - Aggregation myAgg = (Aggregation) agg; - myAgg.isNull = true; - } - - @Override - public Object evaluateOutput( - AggregationBuffer agg) throws HiveException { - Aggregation myagg = (Aggregation) agg; - if (myagg.isNull) { - return null; - } - else { - result.set(myagg.value); - return result; - } - } - - @Override - public ObjectInspector getOutputObjectInspector() { - return ; - } -} - diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFSum.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFSum.txt deleted file mode 100644 index aaaa6ad..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFSum.txt +++ /dev/null @@ -1,409 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen; - -import java.util.ArrayList; - -import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates. - VectorAggregateExpression.AggregationBuffer; -import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.hive.serde2.io.DoubleWritable; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; - -/** -* . Vectorized implementation for SUM aggregates. -*/ -@Description(name = "sum", value = "_FUNC_(expr) - Returns the sum value of expr (vectorized, type: )") -public class extends VectorAggregateExpression { - - /** - /* class for storing the current aggregate value. - */ - private static final class Aggregation implements AggregationBuffer { - sum; - boolean isNull; - - public void sumValue( value) { - if (isNull) { - sum = value; - isNull = false; - } else { - sum += value; - } - } - } - - VectorExpression inputExpression; - private result; - - public (VectorExpression inputExpression) { - super(); - this.inputExpression = inputExpression; - result = new (); - } - - private Aggregation getCurrentAggregationBuffer( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - int row) { - VectorAggregationBufferRow mySet = aggregationBufferSets[row]; - Aggregation myagg = (Aggregation) mySet.getAggregationBuffer(aggregateIndex); - return myagg; - } - - @Override - public void aggregateInputSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - VectorizedRowBatch batch) throws HiveException { - - int batchSize = batch.size; - - if (batchSize == 0) { - return; - } - - inputExpression.evaluate(batch); - - LongColumnVector inputVector = (LongColumnVector)batch. - cols[this.inputExpression.getOutputColumn()]; - long[] vector = inputVector.vector; - - if (inputVector.noNulls) { - if (inputVector.isRepeating) { - iterateNoNullsRepeatingWithAggregationSelection( - aggregationBufferSets, aggregateIndex, - vector[0], batchSize); - } else { - if (batch.selectedInUse) { - iterateNoNullsSelectionWithAggregationSelection( - aggregationBufferSets, aggregateIndex, - vector, batch.selected, batchSize); - } else { - iterateNoNullsWithAggregationSelection( - aggregationBufferSets, aggregateIndex, - vector, batchSize); - } - } - } else { - if (inputVector.isRepeating) { - if (batch.selectedInUse) { - iterateHasNullsRepeatingSelectionWithAggregationSelection( - aggregationBufferSets, aggregateIndex, - vector[0], batchSize, batch.selected, inputVector.isNull); - } else { - iterateHasNullsRepeatingWithAggregationSelection( - aggregationBufferSets, aggregateIndex, - vector[0], batchSize, inputVector.isNull); - } - } else { - if (batch.selectedInUse) { - iterateHasNullsSelectionWithAggregationSelection( - aggregationBufferSets, aggregateIndex, - vector, batchSize, batch.selected, inputVector.isNull); - } else { - iterateHasNullsWithAggregationSelection( - aggregationBufferSets, aggregateIndex, - vector, batchSize, inputVector.isNull); - } - } - } - } - - private void iterateNoNullsRepeatingWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - long value, - int batchSize) { - - for (int i=0; i < batchSize; ++i) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregateIndex, - i); - myagg.sumValue(value); - } - } - - private void iterateNoNullsSelectionWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - long[] values, - int[] selection, - int batchSize) { - - for (int i=0; i < batchSize; ++i) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregateIndex, - i); - myagg.sumValue(values[selection[i]]); - } - } - - private void iterateNoNullsWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - long[] values, - int batchSize) { - for (int i=0; i < batchSize; ++i) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregateIndex, - i); - myagg.sumValue(values[i]); - } - } - - private void iterateHasNullsRepeatingSelectionWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - long value, - int batchSize, - int[] selection, - boolean[] isNull) { - - for (int i=0; i < batchSize; ++i) { - if (!isNull[selection[i]]) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregateIndex, - i); - myagg.sumValue(value); - } - } - - } - - private void iterateHasNullsRepeatingWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - long value, - int batchSize, - boolean[] isNull) { - - for (int i=0; i < batchSize; ++i) { - if (!isNull[i]) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregateIndex, - i); - myagg.sumValue(value); - } - } - } - - private void iterateHasNullsSelectionWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - long[] values, - int batchSize, - int[] selection, - boolean[] isNull) { - - for (int j=0; j < batchSize; ++j) { - int i = selection[j]; - if (!isNull[i]) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregateIndex, - j); - myagg.sumValue(values[i]); - } - } - } - - private void iterateHasNullsWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - long[] values, - int batchSize, - boolean[] isNull) { - - for (int i=0; i < batchSize; ++i) { - if (!isNull[i]) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregateIndex, - i); - myagg.sumValue(values[i]); - } - } - } - - - @Override - public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) - throws HiveException { - - inputExpression.evaluate(batch); - - inputVector = ()batch. - cols[this.inputExpression.getOutputColumn()]; - - int batchSize = batch.size; - - if (batchSize == 0) { - return; - } - - Aggregation myagg = (Aggregation)agg; - - [] vector = inputVector.vector; - - if (inputVector.isRepeating) { - if (inputVector.noNulls || !inputVector.isNull[0]) { - if (myagg.isNull) { - myagg.isNull = false; - myagg.sum = 0; - } - myagg.sum += vector[0]*batchSize; - } - return; - } - - if (!batch.selectedInUse && inputVector.noNulls) { - iterateNoSelectionNoNulls(myagg, vector, batchSize); - } - else if (!batch.selectedInUse) { - iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull); - } - else if (inputVector.noNulls){ - iterateSelectionNoNulls(myagg, vector, batchSize, batch.selected); - } - else { - iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, batch.selected); - } - } - - private void iterateSelectionHasNulls( - Aggregation myagg, - [] vector, - int batchSize, - boolean[] isNull, - int[] selected) { - - for (int j=0; j< batchSize; ++j) { - int i = selected[j]; - if (!isNull[i]) { - value = vector[i]; - if (myagg.isNull) { - myagg.isNull = false; - myagg.sum = 0; - } - myagg.sum += value; - } - } - } - - private void iterateSelectionNoNulls( - Aggregation myagg, - [] vector, - int batchSize, - int[] selected) { - - if (myagg.isNull) { - myagg.sum = 0; - myagg.isNull = false; - } - - for (int i=0; i< batchSize; ++i) { - value = vector[selected[i]]; - myagg.sum += value; - } - } - - private void iterateNoSelectionHasNulls( - Aggregation myagg, - [] vector, - int batchSize, - boolean[] isNull) { - - for(int i=0;i value = vector[i]; - if (myagg.isNull) { - myagg.sum = 0; - myagg.isNull = false; - } - myagg.sum += value; - } - } - } - - private void iterateNoSelectionNoNulls( - Aggregation myagg, - [] vector, - int batchSize) { - if (myagg.isNull) { - myagg.sum = 0; - myagg.isNull = false; - } - - for (int i=0;i value = vector[i]; - myagg.sum += value; - } - } - - @Override - public AggregationBuffer getNewAggregationBuffer() throws HiveException { - return new Aggregation(); - } - - @Override - public void reset(AggregationBuffer agg) throws HiveException { - Aggregation myAgg = (Aggregation) agg; - myAgg.isNull = true; - } - - @Override - public Object evaluateOutput(AggregationBuffer agg) throws HiveException { - Aggregation myagg = (Aggregation) agg; - if (myagg.isNull) { - return null; - } - else { - result.set(myagg.sum); - return result; - } - } - - @Override - public ObjectInspector getOutputObjectInspector() { - return ; - } -} - diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt deleted file mode 100644 index daae57b..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt +++ /dev/null @@ -1,487 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen; - -import java.util.ArrayList; - -import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates - .VectorAggregateExpression.AggregationBuffer; -import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.hive.serde2.io.DoubleWritable; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; - -/** -* . Vectorized implementation for VARIANCE aggregates. -*/ -@Description(name = "", value = "") -public class extends VectorAggregateExpression { - - /** - /* class for storing the current aggregate value. - */ - private static final class Aggregation implements AggregationBuffer { - double sum; - long count; - double variance; - boolean isNull; - - public void init() { - isNull = false; - sum = 0; - count = 0; - variance = 0; - } - } - - private VectorExpression inputExpression; - private LongWritable resultCount; - private DoubleWritable resultSum; - private DoubleWritable resultVariance; - private Object[] partialResult; - - private ObjectInspector soi; - - - public (VectorExpression inputExpression) { - super(); - this.inputExpression = inputExpression; - partialResult = new Object[3]; - resultCount = new LongWritable(); - resultSum = new DoubleWritable(); - resultVariance = new DoubleWritable(); - partialResult[0] = resultCount; - partialResult[1] = resultSum; - partialResult[2] = resultVariance; - initPartialResultInspector(); - } - - private void initPartialResultInspector() { - ArrayList foi = new ArrayList(); - foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector); - foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); - foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); - - ArrayList fname = new ArrayList(); - fname.add("count"); - fname.add("sum"); - fname.add("variance"); - - soi = ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi); - } - - private Aggregation getCurrentAggregationBuffer( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - int row) { - VectorAggregationBufferRow mySet = aggregationBufferSets[row]; - Aggregation myagg = (Aggregation) mySet.getAggregationBuffer(aggregateIndex); - return myagg; - } - - - @Override - public void aggregateInputSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - VectorizedRowBatch batch) throws HiveException { - - inputExpression.evaluate(batch); - - inputVector = ()batch. - cols[this.inputExpression.getOutputColumn()]; - - int batchSize = batch.size; - - if (batchSize == 0) { - return; - } - - [] vector = inputVector.vector; - - if (inputVector.isRepeating) { - if (inputVector.noNulls || !inputVector.isNull[0]) { - iterateRepeatingNoNullsWithAggregationSelection( - aggregationBufferSets, aggregateIndex, vector[0], batchSize); - } - } - else if (!batch.selectedInUse && inputVector.noNulls) { - iterateNoSelectionNoNullsWithAggregationSelection( - aggregationBufferSets, aggregateIndex, vector, batchSize); - } - else if (!batch.selectedInUse) { - iterateNoSelectionHasNullsWithAggregationSelection( - aggregationBufferSets, aggregateIndex, vector, batchSize, inputVector.isNull); - } - else if (inputVector.noNulls){ - iterateSelectionNoNullsWithAggregationSelection( - aggregationBufferSets, aggregateIndex, vector, batchSize, batch.selected); - } - else { - iterateSelectionHasNullsWithAggregationSelection( - aggregationBufferSets, aggregateIndex, vector, batchSize, - inputVector.isNull, batch.selected); - } - - } - - private void iterateRepeatingNoNullsWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - value, - int batchSize) { - - for (int i=0; i 1) { - double t = myagg.count*value - myagg.sum; - myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); - } - } - } - - private void iterateSelectionHasNullsWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - [] vector, - int batchSize, - boolean[] isNull, - int[] selected) { - - for (int j=0; j< batchSize; ++j) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregateIndex, - j); - int i = selected[j]; - if (!isNull[i]) { - value = vector[i]; - if (myagg.isNull) { - myagg.init (); - } - myagg.sum += value; - myagg.count += 1; - if(myagg.count > 1) { - double t = myagg.count*value - myagg.sum; - myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); - } - } - } - } - - private void iterateSelectionNoNullsWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - [] vector, - int batchSize, - int[] selected) { - - for (int i=0; i< batchSize; ++i) { - Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, - aggregateIndex, - i); - value = vector[selected[i]]; - if (myagg.isNull) { - myagg.init (); - } - myagg.sum += value; - myagg.count += 1; - if(myagg.count > 1) { - double t = myagg.count*value - myagg.sum; - myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); - } - } - } - - private void iterateNoSelectionHasNullsWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - [] vector, - int batchSize, - boolean[] isNull) { - - for(int i=0;i value = vector[i]; - if (myagg.isNull) { - myagg.init (); - } - myagg.sum += value; - myagg.count += 1; - if(myagg.count > 1) { - double t = myagg.count*value - myagg.sum; - myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); - } - } - } - } - - private void iterateNoSelectionNoNullsWithAggregationSelection( - VectorAggregationBufferRow[] aggregationBufferSets, - int aggregateIndex, - [] vector, - int batchSize) { - - for (int i=0; i value = vector[i]; - myagg.sum += value; - myagg.count += 1; - if(myagg.count > 1) { - double t = myagg.count*value - myagg.sum; - myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); - } - } - } - - @Override - public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) - throws HiveException { - - inputExpression.evaluate(batch); - - inputVector = ()batch. - cols[this.inputExpression.getOutputColumn()]; - - int batchSize = batch.size; - - if (batchSize == 0) { - return; - } - - Aggregation myagg = (Aggregation)agg; - - [] vector = inputVector.vector; - - if (inputVector.isRepeating) { - if (inputVector.noNulls || !inputVector.isNull[0]) { - iterateRepeatingNoNulls(myagg, vector[0], batchSize); - } - } - else if (!batch.selectedInUse && inputVector.noNulls) { - iterateNoSelectionNoNulls(myagg, vector, batchSize); - } - else if (!batch.selectedInUse) { - iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull); - } - else if (inputVector.noNulls){ - iterateSelectionNoNulls(myagg, vector, batchSize, batch.selected); - } - else { - iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, batch.selected); - } - } - - private void iterateRepeatingNoNulls( - Aggregation myagg, - value, - int batchSize) { - - if (myagg.isNull) { - myagg.init (); - } - - // TODO: conjure a formula w/o iterating - // - - myagg.sum += value; - myagg.count += 1; - if(myagg.count > 1) { - double t = myagg.count*value - myagg.sum; - myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); - } - - // We pulled out i=0 so we can remove the count > 1 check in the loop - for (int i=1; i[] vector, - int batchSize, - boolean[] isNull, - int[] selected) { - - for (int j=0; j< batchSize; ++j) { - int i = selected[j]; - if (!isNull[i]) { - value = vector[i]; - if (myagg.isNull) { - myagg.init (); - } - myagg.sum += value; - myagg.count += 1; - if(myagg.count > 1) { - double t = myagg.count*value - myagg.sum; - myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); - } - } - } - } - - private void iterateSelectionNoNulls( - Aggregation myagg, - [] vector, - int batchSize, - int[] selected) { - - if (myagg.isNull) { - myagg.init (); - } - - value = vector[selected[0]]; - myagg.sum += value; - myagg.count += 1; - if(myagg.count > 1) { - double t = myagg.count*value - myagg.sum; - myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); - } - - // i=0 was pulled out to remove the count > 1 check in the loop - // - for (int i=1; i< batchSize; ++i) { - value = vector[selected[i]]; - myagg.sum += value; - myagg.count += 1; - double t = myagg.count*value - myagg.sum; - myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); - } - } - - private void iterateNoSelectionHasNulls( - Aggregation myagg, - [] vector, - int batchSize, - boolean[] isNull) { - - for(int i=0;i value = vector[i]; - if (myagg.isNull) { - myagg.init (); - } - myagg.sum += value; - myagg.count += 1; - if(myagg.count > 1) { - double t = myagg.count*value - myagg.sum; - myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); - } - } - } - } - - private void iterateNoSelectionNoNulls( - Aggregation myagg, - [] vector, - int batchSize) { - - if (myagg.isNull) { - myagg.init (); - } - - value = vector[0]; - myagg.sum += value; - myagg.count += 1; - - if(myagg.count > 1) { - double t = myagg.count*value - myagg.sum; - myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1)); - } - - // i=0 was pulled out to remove count > 1 check - for (int i=1; i