diff --git ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java index a15eca0..86e8dfb 100644 --- ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java +++ ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java @@ -444,8 +444,10 @@ private String buildDir; private String expressionOutputDirectory; + private String expressionClassesDirectory; private String expressionTemplateDirectory; private String udafOutputDirectory; + private String udafClassesDirectory; private String udafTemplateDirectory; private GenVectorTestCode testCodeGen; @@ -461,17 +463,24 @@ public void init(String templateBaseDir, String buildDir) { File generationDirectory = new File(templateBaseDir); String buildPath = joinPath(buildDir, "generated-sources", "java"); + String compiledPath = joinPath(buildDir, "classes"); - File exprOutput = new File(joinPath(buildPath, "org", "apache", "hadoop", - "hive", "ql", "exec", "vector", "expressions", "gen")); + String expression = joinPath("org", "apache", "hadoop", + "hive", "ql", "exec", "vector", "expressions", "gen"); + File exprOutput = new File(joinPath(buildPath, expression)); + File exprClasses = new File(joinPath(compiledPath, expression)); expressionOutputDirectory = exprOutput.getAbsolutePath(); + expressionClassesDirectory = exprClasses.getAbsolutePath(); expressionTemplateDirectory = joinPath(generationDirectory.getAbsolutePath(), "ExpressionTemplates"); - File udafOutput = new File(joinPath(buildPath, "org", "apache", "hadoop", - "hive", "ql", "exec", "vector", "expressions", "aggregates", "gen")); + String udaf = joinPath("org", "apache", "hadoop", + "hive", "ql", "exec", "vector", "expressions", "aggregates", "gen"); + File udafOutput = new File(joinPath(buildPath, udaf)); + File udafClasses = new File(joinPath(compiledPath, udaf)); udafOutputDirectory = udafOutput.getAbsolutePath(); + udafClassesDirectory = udafClasses.getAbsolutePath(); udafTemplateDirectory = joinPath(generationDirectory.getAbsolutePath(), "UDAFTemplates"); @@ -502,7 +511,7 @@ public void execute() throws BuildException { try { this.generate(); } catch (Exception e) { - new BuildException(e); + throw new BuildException(e); } } @@ -569,14 +578,14 @@ private void generateFilterStringColumnBetween(String[] tdesc) throws IOExceptio String optionalNot = tdesc[1]; String className = "FilterStringColumn" + (optionalNot.equals("!") ? "Not" : "") + "Between"; - String outputFile = joinPath(this.expressionOutputDirectory, className + ".java"); - - // Read the template into a string, expand it, and write it. - String templateFile = joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"); + // Read the template into a string, expand it, and write it. + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("", className); templateString = templateString.replaceAll("", optionalNot); - writeFile(outputFile, templateString); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); } private void generateFilterColumnBetween(String[] tdesc) throws IOException { @@ -586,17 +595,17 @@ private void generateFilterColumnBetween(String[] tdesc) throws IOException { String className = "Filter" + getCamelCaseType(operandType) + "Column" + (optionalNot.equals("!") ? "Not" : "") + "Between"; String inputColumnVectorType = getColumnVectorType(operandType); - String outputFile = joinPath(this.expressionOutputDirectory, className + ".java"); // Read the template into a string, expand it, and write it. - String templateFile = joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"); + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("", className); templateString = templateString.replaceAll("", inputColumnVectorType); templateString = templateString.replaceAll("", operandType); templateString = templateString.replaceAll("", optionalNot); - writeFile(outputFile, templateString); + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); } private void generateColumnCompareColumn(String[] tdesc) throws IOException { @@ -620,8 +629,7 @@ private void generateVectorUDAFMinMax(String[] tdesc) throws Exception { String writableType = getOutputWritableType(valueType); String inspectorType = getOutputObjectInspector(valueType); - String outputFile = joinPath(this.udafOutputDirectory, className + ".java"); - String templateFile = joinPath(this.udafTemplateDirectory, tdesc[0] + ".txt"); + File templateFile = new File(joinPath(this.udafTemplateDirectory, tdesc[0] + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("", className); @@ -632,8 +640,8 @@ private void generateVectorUDAFMinMax(String[] tdesc) throws Exception { templateString = templateString.replaceAll("", descValue); templateString = templateString.replaceAll("", writableType); templateString = templateString.replaceAll("", inspectorType); - writeFile(outputFile, templateString); - + writeFile(templateFile.lastModified(), udafOutputDirectory, udafClassesDirectory, + className, templateString); } private void generateVectorUDAFMinMaxString(String[] tdesc) throws Exception { @@ -642,16 +650,15 @@ private void generateVectorUDAFMinMaxString(String[] tdesc) throws Exception { String descName = tdesc[3]; String descValue = tdesc[4]; - String outputFile = joinPath(this.udafOutputDirectory, className + ".java"); - String templateFile = joinPath(this.udafTemplateDirectory, tdesc[0] + ".txt"); + File templateFile = new File(joinPath(this.udafTemplateDirectory, tdesc[0] + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("", className); templateString = templateString.replaceAll("", operatorSymbol); templateString = templateString.replaceAll("", descName); templateString = templateString.replaceAll("", descValue); - writeFile(outputFile, templateString); - + writeFile(templateFile.lastModified(), udafOutputDirectory, udafClassesDirectory, + className, templateString); } private void generateVectorUDAFSum(String[] tdesc) throws Exception { @@ -662,8 +669,7 @@ private void generateVectorUDAFSum(String[] tdesc) throws Exception { String writableType = getOutputWritableType(valueType); String inspectorType = getOutputObjectInspector(valueType); - String outputFile = joinPath(this.udafOutputDirectory, className + ".java"); - String templateFile = joinPath(this.udafTemplateDirectory, tdesc[0] + ".txt"); + File templateFile = new File(joinPath(this.udafTemplateDirectory, tdesc[0] + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("", className); @@ -671,7 +677,8 @@ private void generateVectorUDAFSum(String[] tdesc) throws Exception { templateString = templateString.replaceAll("", columnType); templateString = templateString.replaceAll("", writableType); templateString = templateString.replaceAll("", inspectorType); - writeFile(outputFile, templateString); + writeFile(templateFile.lastModified(), udafOutputDirectory, udafClassesDirectory, + className, templateString); } private void generateVectorUDAFAvg(String[] tdesc) throws IOException { @@ -679,14 +686,14 @@ private void generateVectorUDAFAvg(String[] tdesc) throws IOException { String valueType = tdesc[2]; String columnType = getColumnVectorType(valueType); - String outputFile = joinPath(this.udafOutputDirectory, className + ".java"); - String templateFile = joinPath(this.udafTemplateDirectory, tdesc[0] + ".txt"); + File templateFile = new File(joinPath(this.udafTemplateDirectory, tdesc[0] + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("", className); templateString = templateString.replaceAll("", valueType); templateString = templateString.replaceAll("", columnType); - writeFile(outputFile, templateString); + writeFile(templateFile.lastModified(), udafOutputDirectory, udafClassesDirectory, + className, templateString); } private void generateVectorUDAFVar(String[] tdesc) throws IOException { @@ -697,8 +704,7 @@ private void generateVectorUDAFVar(String[] tdesc) throws IOException { String descriptionValue = tdesc[5]; String columnType = getColumnVectorType(valueType); - String outputFile = joinPath(this.udafOutputDirectory, className + ".java"); - String templateFile = joinPath(this.udafTemplateDirectory, tdesc[0] + ".txt"); + File templateFile = new File(joinPath(this.udafTemplateDirectory, tdesc[0] + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("", className); @@ -707,7 +713,8 @@ private void generateVectorUDAFVar(String[] tdesc) throws IOException { templateString = templateString.replaceAll("", varianceFormula); templateString = templateString.replaceAll("", descriptionName); templateString = templateString.replaceAll("", descriptionValue); - writeFile(outputFile, templateString); + writeFile(templateFile.lastModified(), udafOutputDirectory, udafClassesDirectory, + className, templateString); } private void generateFilterStringScalarCompareColumn(String[] tdesc) throws IOException { @@ -752,15 +759,15 @@ private void generateStringColumnCompareColumn(String[] tdesc) throws IOExceptio private void generateStringColumnCompareScalar(String[] tdesc, String className) throws IOException { - String operatorSymbol = tdesc[2]; - String outputFile = joinPath(this.expressionOutputDirectory, className + ".java"); - // Read the template into a string; - String templateFile = joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"); - String templateString = readFile(templateFile); - // Expand, and write result - templateString = templateString.replaceAll("", className); - templateString = templateString.replaceAll("", operatorSymbol); - writeFile(outputFile, templateString); + String operatorSymbol = tdesc[2]; + // Read the template into a string; + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + // Expand, and write result + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", operatorSymbol); + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); } private void generateFilterColumnCompareColumn(String[] tdesc) throws IOException { @@ -780,8 +787,7 @@ private void generateColumnUnaryMinus(String[] tdesc) throws IOException { String outputColumnVectorType = inputColumnVectorType; String returnType = operandType; String className = getCamelCaseType(operandType) + "ColUnaryMinus"; - String outputFile = joinPath(this.expressionOutputDirectory, className + ".java"); - String templateFile = joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"); + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); String templateString = readFile(templateFile); // Expand, and write result templateString = templateString.replaceAll("", className); @@ -789,7 +795,8 @@ private void generateColumnUnaryMinus(String[] tdesc) throws IOException { templateString = templateString.replaceAll("", outputColumnVectorType); templateString = templateString.replaceAll("", operandType); templateString = templateString.replaceAll("", returnType); - writeFile(outputFile, templateString); + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); } // template, , , , , , @@ -801,8 +808,7 @@ private void generateColumnUnaryFunc(String[] tdesc) throws IOException { String outputColumnVectorType = this.getColumnVectorType(returnType); String className = classNamePrefix + getCamelCaseType(operandType) + "To" + getCamelCaseType(returnType); - String outputFile = joinPath(this.expressionOutputDirectory, className + ".java"); - String templateFile = joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"); + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); String templateString = readFile(templateFile); String funcName = tdesc[4]; String operandCast = tdesc[5]; @@ -818,7 +824,8 @@ private void generateColumnUnaryFunc(String[] tdesc) throws IOException { templateString = templateString.replaceAll("", operandCast); templateString = templateString.replaceAll("", resultCast); templateString = templateString.replaceAll("", cleanup); - writeFile(outputFile, templateString); + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); } private void generateColumnArithmeticColumn(String [] tdesc) throws IOException { @@ -880,10 +887,9 @@ private void generateColumnBinaryOperatorColumn(String[] tdesc, String returnTyp String inputColumnVectorType1 = this.getColumnVectorType(operandType1); String inputColumnVectorType2 = this.getColumnVectorType(operandType2); String operatorSymbol = tdesc[4]; - String outputFile = joinPath(this.expressionOutputDirectory, className + ".java"); //Read the template into a string; - String templateFile = joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"); + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("", className); templateString = templateString.replaceAll("", inputColumnVectorType1); @@ -894,7 +900,8 @@ private void generateColumnBinaryOperatorColumn(String[] tdesc, String returnTyp templateString = templateString.replaceAll("", operandType2); templateString = templateString.replaceAll("", returnType); templateString = templateString.replaceAll("", getCamelCaseType(returnType)); - writeFile(outputFile, templateString); + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); if(returnType==null){ testCodeGen.addColumnColumnFilterTestCases( @@ -918,10 +925,9 @@ private void generateColumnBinaryOperatorScalar(String[] tdesc, String returnTyp String outputColumnVectorType = this.getColumnVectorType(returnType); String inputColumnVectorType = this.getColumnVectorType(operandType1); String operatorSymbol = tdesc[4]; - String outputFile = joinPath(this.expressionOutputDirectory, className + ".java"); //Read the template into a string; - String templateFile = joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"); + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("", className); templateString = templateString.replaceAll("", inputColumnVectorType); @@ -930,7 +936,8 @@ private void generateColumnBinaryOperatorScalar(String[] tdesc, String returnTyp templateString = templateString.replaceAll("", operandType1); templateString = templateString.replaceAll("", operandType2); templateString = templateString.replaceAll("", returnType); - writeFile(outputFile, templateString); + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); if(returnType==null) { testCodeGen.addColumnScalarFilterTestCases( @@ -956,10 +963,9 @@ private void generateScalarBinaryOperatorColumn(String[] tdesc, String returnTyp String outputColumnVectorType = this.getColumnVectorType(returnType); String inputColumnVectorType = this.getColumnVectorType(operandType2); String operatorSymbol = tdesc[4]; - String outputFile = joinPath(this.expressionOutputDirectory, className + ".java"); //Read the template into a string; - String templateFile = joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"); + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("", className); templateString = templateString.replaceAll("", inputColumnVectorType); @@ -968,7 +974,8 @@ private void generateScalarBinaryOperatorColumn(String[] tdesc, String returnTyp templateString = templateString.replaceAll("", operandType1); templateString = templateString.replaceAll("", operandType2); templateString = templateString.replaceAll("", returnType); - writeFile(outputFile, templateString); + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); if(returnType==null) { testCodeGen.addColumnScalarFilterTestCases( @@ -1009,13 +1016,29 @@ private void generateScalarArithmeticColumn(String[] tdesc) throws IOException { } - static void writeFile(String outputFile, String str) throws IOException { + static void writeFile(long templateTime, String outputDir, String classedDir, String name, String str) + throws IOException { + File outputFile = new File(outputDir, name + ".java"); + File outputClass = new File(classedDir, name + ".class"); + if (outputFile.lastModified() > templateTime && outputFile.length() == str.length() && + outputClass.lastModified() > templateTime) { + // best effort + return; + } + writeFile(str, outputFile); + } + + static void writeFile(String str, File outputFile) throws IOException { BufferedWriter w = new BufferedWriter(new FileWriter(outputFile)); w.write(str); w.close(); } - static String readFile(String templateFile) throws IOException { + static String readFile(String templateFile) throws IOException { + return readFile(new File(templateFile)); + } + + static String readFile(File templateFile) throws IOException { BufferedReader r = new BufferedReader(new FileReader(templateFile)); String line = r.readLine(); StringBuilder b = new StringBuilder(); diff --git ant/src/org/apache/hadoop/hive/ant/GenVectorTestCode.java ant/src/org/apache/hadoop/hive/ant/GenVectorTestCode.java index ef5dcb9..94a19d1 100644 --- ant/src/org/apache/hadoop/hive/ant/GenVectorTestCode.java +++ ant/src/org/apache/hadoop/hive/ant/GenVectorTestCode.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ant; +import java.io.File; import java.io.IOException; import java.util.HashMap; @@ -235,7 +236,7 @@ public void generateTestSuites() throws IOException { String outputFile = GenVectorCode.joinPath(this.testOutputDir, testClass + ".java"); - GenVectorCode.writeFile(outputFile, templateString); + GenVectorCode.writeFile(outputFile, new File(templateString)); } }