From eef719617fed9615f5af72d83d7656d32b3fcf69 Mon Sep 17 00:00:00 2001 From: manukranthk Date: Thu, 23 Oct 2014 17:35:42 -0700 Subject: [PATCH] HBASE-12333 Add Integration Test Runner which is more friendly --- .../hbase/IntegrationTestRunnerWithJSONOutput.java | 143 +++++++++++++++++++++ .../hadoop/hbase/IntegrationTestsDriver.java | 2 +- .../hbase/test/IntegrationTestBigLinkedList.java | 27 +++- ...IntegrationTestBigLinkedListWithVisibility.java | 11 +- 4 files changed, 172 insertions(+), 11 deletions(-) create mode 100644 hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestRunnerWithJSONOutput.java diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestRunnerWithJSONOutput.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestRunnerWithJSONOutput.java new file mode 100644 index 0000000..5dd19e1 --- /dev/null +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestRunnerWithJSONOutput.java @@ -0,0 +1,143 @@ +package org.apache.hadoop.hbase; + +import java.io.File; +import java.io.FileWriter; +import java.io.PrintStream; +import java.nio.file.Files; +import java.nio.file.OpenOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileAttribute; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.test.IntegrationTestBigLinkedList; +import org.apache.hadoop.util.ToolRunner; +import org.codehaus.jettison.json.JSONWriter; +import org.junit.internal.TextListener; +import org.junit.runner.JUnitCore; +import org.junit.runner.Result; +import org.junit.runner.notification.Failure; + +/** + * This is a simple driver which runs a list of well know Integration tests listed below with + * parameters passed in through the command line. + * + * IntegrationTestBigLinkedList#Loop + * IntegrationTestBigLinkedListWithVisibility#Loop + * + * + */ +public class IntegrationTestRunnerWithJSONOutput extends IntegrationTestsDriver { + private static final Log LOG = LogFactory.getLog(IntegrationTestRunnerWithJSONOutput.class); + private static final String JOB_NAME_KEY = "hbase.it.job.name"; + private static final String IT_RESULTS_DIR_KEY = "hbase.it.results.dir"; + private String confFile = null; + + @Override + protected void addOptions() { + super.addOptions(); + addOptWithArg("D", + "Configuration parameter." + + " Note that some configuration parameters might not make it to the task trackers"); + addOptWithArg("c", "conf", + "Configuration file that should be used for this test runner. Any parameters overriden by " + + "command line will take effect"); + } + + @Override + protected void processOptions(CommandLine cmd) { + if (cmd.hasOption("D")) { + String [] confProperties = cmd.getOptionValues("D"); + for (String property : confProperties) { + try { + String [] parts = property.split("="); + if (parts.length != 2) throw new IllegalArgumentException( + "Expected foramt of conf properties is =, but got this : " + property); + conf.set(parts[0], parts[1]); + } catch (Exception e) { + LOG.error("Unsupported conf params passed.", e); + } + } + } + if (cmd.hasOption("c")) { + this.confFile = cmd.getOptionValue("c"); + } + } + + @Override + protected int doWork() throws Exception { + // In case we provide a configuration file on the command line, it will be applied. + if (confFile != null) { + conf.addResource(confFile); + } + //this is called from the command line, so we should set to use the distributed cluster + IntegrationTestingUtility.setUseDistributedCluster(conf); + String jobName = conf.get(JOB_NAME_KEY); + String resultsDir = conf.get(IT_RESULTS_DIR_KEY); + + String jsonFile = "/tmp/" + jobName + ".json"; + FileWriter jsonFileWriter = new FileWriter(jsonFile, true); + + JSONWriter jsonWriter = new JSONWriter(jsonFileWriter); + jsonWriter.object(); // { + + Class[] classes = findIntegrationTestClasses(); + LOG.info("Found " + classes.length + " integration tests to run:"); + FileSystem fs = FileSystem.get(conf); + boolean successful = true; + for (Class aClass : classes) { + LOG.info("Running : " + aClass); + JUnitCore junit = new JUnitCore(); + String outFile = "/tmp/" + aClass.getName() + ".out"; + junit.addListener(new TextListener(new PrintStream(new File(outFile)))); + + Result result = junit.run(aClass); + Path outFileOnDFS = new Path(new Path(resultsDir, jobName), aClass.getName() + ".out"); + + fs.copyFromLocalFile(true, new Path(outFile), outFileOnDFS); + + jsonWriter.key(aClass.getName()); // aClass.getName(): + jsonWriter.object(); // { + jsonWriter.key("outfile"); // outfile: + jsonWriter.value(outFileOnDFS.getName()); // filePath + jsonWriter.key("success"); // success: + jsonWriter.value(result.wasSuccessful()); // result.wasSuccessful() + jsonWriter.key("failureCnt"); // failureCnt: + jsonWriter.value(result.getFailureCount()); // result.getFailureCount() + jsonWriter.key("runTime"); // runTime: + jsonWriter.value(result.getRunTime()); // result.getRunTime() + jsonWriter.key("failures"); // runTime: + jsonWriter.value(getFailuresString(result)); // result.getRunTime() + jsonWriter.endObject(); // } + successful &= result.wasSuccessful(); + } + jsonWriter.endObject(); // } + jsonFileWriter.close(); + fs.copyFromLocalFile(true, new Path(jsonFile), + new Path(new Path(resultsDir, jobName), jobName + ".json")); + + return successful ? 0 : 1; + } + + private String getFailuresString(Result result) { + StringBuilder sb = new StringBuilder(); + for (Failure f : result.getFailures()) { + sb.append("Message : "); + sb.append(f.getMessage()); + sb.append(" Stack Trace : "); + sb.append(f.getTrace()); + } + return sb.toString(); + } + + public static void main(String[] args) throws Exception { + Configuration conf = HBaseConfiguration.create(); + IntegrationTestingUtility.setUseDistributedCluster(conf); + int ret = ToolRunner.run(conf, new IntegrationTestRunnerWithJSONOutput(), args); + System.exit(ret); + } +} diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java index 47ce9e1..505d4d9 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java @@ -88,7 +88,7 @@ public class IntegrationTestsDriver extends AbstractHBaseTool { * Returns test classes annotated with @Category(IntegrationTests.class), * according to the filter specific on the command line (if any). */ - private Class[] findIntegrationTestClasses() + protected Class[] findIntegrationTestClasses() throws ClassNotFoundException, LinkageError, IOException { ClassTestFinder.TestFileNameFilter nameFilter = new ClassTestFinder.TestFileNameFilter(); ClassFinder classFinder = new ClassFinder(nameFilter, nameFilter, intTestFilter); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java index 177341f..a0b5d5b 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java @@ -182,18 +182,27 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { protected static final byte[] COLUMN_COUNT = Bytes.toBytes("count"); /** How many rows to write per map task. This has to be a multiple of 25M */ - private static final String GENERATOR_NUM_ROWS_PER_MAP_KEY + protected static final String GENERATOR_NUM_ROWS_PER_MAP_KEY = "IntegrationTestBigLinkedList.generator.num_rows"; - private static final String GENERATOR_NUM_MAPPERS_KEY + protected static final String GENERATOR_NUM_MAPPERS_KEY = "IntegrationTestBigLinkedList.generator.map.tasks"; + + protected static final String VERIFY_NUM_REDUCERS_KEY = + "IntegrationTestBigLinkedList.verify.reducer.tasks"; - private static final String GENERATOR_WIDTH_KEY + protected static final String GENERATOR_WIDTH_KEY = "IntegrationTestBigLinkedList.generator.width"; - private static final String GENERATOR_WRAP_KEY + protected static final String GENERATOR_WRAP_KEY = "IntegrationTestBigLinkedList.generator.wrap"; + protected static final String LOOP_NUM_ITERATIONS_KEY = + "IntegrationTestBigLinkedList.generator.map.tasks"; + + protected static final String ITBLL_OUTPUT_FOLDER_KEY = + "IntegrationTestBigLinkedList.output.folder"; + protected int NUM_SLAVES_BASE = 3; // number of slaves for the cluster private static final int MISSING_ROWS_TO_LOG = 50; @@ -1114,10 +1123,14 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { @Test public void testContinuousIngest() throws IOException, Exception { - //Loop + // Loop int ret = ToolRunner.run(getTestingUtil(getConf()).getConfiguration(), new Loop(), - new String[] {"1", "1", "2000000", - util.getDataTestDirOnTestFS("IntegrationTestBigLinkedList").toString(), "1"}); + new String[] {conf.getInt(LOOP_NUM_ITERATIONS_KEY, 1) + "", + conf.getInt(GENERATOR_NUM_MAPPERS_KEY, 1) + "", + conf.getInt(GENERATOR_NUM_ROWS_PER_MAP_KEY, 2000000) + "", + conf.get(ITBLL_OUTPUT_FOLDER_KEY, + util.getDataTestDirOnTestFS("IntegrationTestBigLinkedList").toString()), + conf.getInt(VERIFY_NUM_REDUCERS_KEY, 1) + ""}); org.junit.Assert.assertEquals(0, ret); } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java index 30ca60d..fa65daf 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java @@ -638,9 +638,14 @@ public class IntegrationTestBigLinkedListWithVisibility extends IntegrationTestB int ret = ToolRunner.run( getTestingUtil(getConf()).getConfiguration(), new VisibilityLoop(), - new String[] { "1", "1", "20000", - util.getDataTestDirOnTestFS("IntegrationTestBigLinkedListWithVisibility").toString(), - "1", "10000" }); + new String[] { + conf.getInt(LOOP_NUM_ITERATIONS_KEY, 1) + "", + conf.getInt(GENERATOR_NUM_MAPPERS_KEY, 1) + "", + conf.getInt(GENERATOR_NUM_ROWS_PER_MAP_KEY, 20000) + "", + conf.get(ITBLL_OUTPUT_FOLDER_KEY, + util.getDataTestDirOnTestFS("IntegrationTestBigLinkedListWithVisibility").toString()), + conf.getInt(VERIFY_NUM_REDUCERS_KEY, 1) + "", + conf.getInt(GENERATOR_WIDTH_KEY, 10000) + ""}); org.junit.Assert.assertEquals(0, ret); } -- 1.9.4