diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java index 3155f08..31f6fa8 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java @@ -21,9 +21,17 @@ import static com.google.common.base.Preconditions.checkNotNull; import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Properties; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; @@ -31,6 +39,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.base.Supplier; +import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -97,6 +106,8 @@ public TestParser(Context context, String testCasePropertyName, return result; } private List parseQFileTests() { + Map properties = parseQTestProperties(); + Splitter splitter = Splitter.on(" ").trimResults().omitEmptyStrings(); List result = Lists.newArrayList(); for(String alias : context.getString("qFileTests", "").split(" ")) { @@ -111,22 +122,25 @@ public TestParser(Context context, String testCasePropertyName, for(String excludedTestGroup : splitter.split(testContext.getString("exclude", ""))) { excludedTests.addAll(Arrays.asList(testContext. getString(Joiner.on(".").join("groups", excludedTestGroup), "").trim().split(" "))); + expandTestProperties(excludedTests, properties); } Set isolatedTests = Sets.newHashSet(); for(String ioslatedTestGroup : splitter.split(testContext.getString("isolate", ""))) { isolatedTests.addAll(Arrays.asList(testContext. getString(Joiner.on(".").join("groups", ioslatedTestGroup), "").trim().split(" "))); + expandTestProperties(isolatedTests, properties); } Set includedTests = Sets.newHashSet(); for(String includedTestGroup : splitter.split(testContext.getString("include", ""))) { includedTests.addAll(Arrays.asList(testContext. getString(Joiner.on(".").join("groups", includedTestGroup), "").trim().split(" "))); + expandTestProperties(includedTests, properties); } - if(!includedTests.isEmpty() && !excludedTests.isEmpty()) { - throw new IllegalArgumentException(String.format("Included and excluded mutally exclusive." + - " Included = %s, excluded = %s", includedTests.toString(), excludedTests.toString())); - } + + //excluded overrides included + includedTests.removeAll(excludedTests); + result.addAll(createQFileTestBatches( driver, checkNotNull(testContext.getString("queryFilesProperty"), "queryFilesProperty").trim(), @@ -175,6 +189,72 @@ public TestParser(Context context, String testCasePropertyName, return testBatches; } + /** + * @return properties loaded from files specified in qFileTests.propertyFiles.${fileName}=${filePath} + */ + private Map parseQTestProperties() { + Map propFiles = context.getSubProperties("qFileTests.propertyFiles."); + Map propertyMap = new HashMap(); + for (String propFile : propFiles.keySet()) { + Properties properties = new Properties(); + String path = sourceDirectory + File.separator + propFiles.get(propFile); + FileInputStream fis = null; + try { + fis = new FileInputStream(path); + properties.load(fis); + } catch (IOException e) { + logger.warn("Error processing Qtest property file", e); + continue; + } finally { + try { + if (fis != null) { + fis.close(); + } + } catch (IOException e) { //ignore + } + } + propertyMap.put(propFile, properties); + logger.info("Loaded Qtest property file: " + path); + } + return propertyMap; + } + + /** + * If any of given tests are of the form: ${fileName}.${property} (test list within a property file), + * then expand them. Then remove those markers from the list of tests. + */ + private void expandTestProperties(Set tests, Map propMap) { + Set toRemove = new HashSet(); + Set toAdd = new HashSet(); + + String pattern = "([^\\.]*)\\.\\$\\{([^}]*)}"; + Pattern r = Pattern.compile(pattern); + for (String test : tests) { + Matcher m = r.matcher(test); + if (m.find()) { + toRemove.add(test); + logger.info("Expanding qfile property: " + test); + String propName = m.group(1); + String propValue = m.group(2); + Properties props = propMap.get(propName); + if (props == null) { + logger.warn("No properties found for : " + propName); + continue; + } + String result = (String) props.get(propValue); + if (result == null || result.isEmpty()) { + logger.warn("No properties found in file: " + propName + " for property: " + propValue); + continue; + } + Iterable splits = Splitter.on(',').trimResults().omitEmptyStrings().split(result); + for (String split : splits) { + toAdd.add(split); + } + } + } + tests.removeAll(toRemove); + tests.addAll(toAdd); + } public Supplier> parse() { return new Supplier>() { diff --git testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestParser.java testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestParser.java index 35680b4..f9ab4c7 100644 --- testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestParser.java +++ testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestParser.java @@ -19,7 +19,10 @@ package org.apache.hive.ptest.execution.conf; import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStream; import java.util.List; +import java.util.Properties; import junit.framework.Assert; @@ -46,6 +49,7 @@ private File unitTestDir1; private File unitTestDir2; private File qFileTestDir; + private File propertyDir; @Before public void setup() throws Exception { @@ -56,6 +60,7 @@ public void setup() throws Exception { "test", "classes"))); unitTestDir2 = Dirs.create(new File(baseDir, Joiner.on("/").join("source", "build", "2", "units", "test", "classes"))); qFileTestDir = Dirs.create(new File(baseDir, Joiner.on("/").join("source", "qfiles"))); + propertyDir = Dirs.create(new File(baseDir, Joiner.on("/").join("source", "props"))); Assert.assertTrue((new File(unitTestDir1, "TestA.class")).createNewFile()); Assert.assertTrue((new File(unitTestDir2, "TestB.class")).createNewFile()); Assert.assertTrue((new File(unitTestDir1, "TestC.class")).createNewFile()); @@ -64,10 +69,20 @@ public void setup() throws Exception { Assert.assertTrue((new File(qFileTestDir, ".svn")).mkdirs()); Assert.assertTrue((new File(qFileTestDir, "dir.q")).mkdirs()); Assert.assertTrue((new File(qFileTestDir, "normal.q")).createNewFile()); + Assert.assertTrue((new File(qFileTestDir, "normal2.q")).createNewFile()); + Assert.assertTrue((new File(qFileTestDir, "normal3.q")).createNewFile()); + Assert.assertTrue((new File(qFileTestDir, "normal4.q")).createNewFile()); Assert.assertTrue((new File(qFileTestDir, "excluded.q")).createNewFile()); Assert.assertTrue((new File(qFileTestDir, "isolated.q")).createNewFile()); Assert.assertTrue((new File(qFileTestDir, "included.q")).createNewFile()); + Properties normalProp = new Properties(); + normalProp.setProperty("normal.one.group", "normal.q,normal2.q"); + normalProp.setProperty("normal.two.group", "normal3.q,normal4.q"); + normalProp.setProperty("excluded.group", "excluded.q"); + normalProp.setProperty("isolated.group", "isolated.q"); + normalProp.setProperty("included.group", "included.q"); + serialize("normal.properties", normalProp); } @After public void teardown() { @@ -107,4 +122,46 @@ public void testParseWithIncludes() throws Exception { List testBatches = testParser.parse().get(); Assert.assertEquals(4, testBatches.size()); } + @Test + public void testParsePropertyFile() throws Exception { + context.put("unitTests.directories", "build/1 build/2"); + context.put("unitTests.include", "TestA TestB"); + context.put("unitTests.isolate", "TestB"); + context.put("qFileTests", "f"); + context.put("qFileTests.propertyFiles.prop", + "props" + File.separator + "normal.properties"); + context.put("qFileTest.f.driver", DRIVER); + context.put("qFileTest.f.directory", "qfiles"); + context.put("qFileTest.f.include", "included"); + context.put("qFileTest.f.isolate", "isolated"); + context.put("qFileTest.f.exclude", "excluded"); + context.put("qFileTest.f.queryFilesProperty", "qfile"); + context.put("qFileTest.f.groups.included", "prop.${normal.one.group} prop.${normal.two.group} prop.${isolated.group}"); + context.put("qFileTest.f.groups.isolated", "prop.${isolated.group}"); + context.put("qFileTest.f.groups.excluded", "prop.${excluded.group}"); + testParser = new TestParser(context, "testcase", workingDirectory, LOG); + List testBatches = testParser.parse().get(); + Assert.assertEquals(4, testBatches.size()); + } + //@Test + public void testParseFile() throws Exception { + File workingDir = new File("../.."); + File testConfigurationFile = new File("conf/trunk-mr2.properties"); + TestConfiguration conf = TestConfiguration.fromFile(testConfigurationFile, LOG); + testParser = new TestParser(conf.getContext(), "test", workingDir, LOG); + List testBatches = testParser.parse().get(); + for (TestBatch testBatch : testBatches) { + System.out.println(testBatch.getTestArguments()); + } + } + + private void serialize(String propFileName, Properties props) throws Exception { + File f = new File(propertyDir, propFileName); + OutputStream out = new FileOutputStream(f); + try { + props.store(out, null); + } finally { + out.close(); + } + } }