From 91f1a6ca9b1cd72ce4d6f8409c07ce16ef1d9fb6 Mon Sep 17 00:00:00 2001 From: Elliott Clark Date: Thu, 15 Aug 2013 16:36:42 -0700 Subject: [PATCH] Clean up IT tests. --- .../org/apache/hadoop/hbase/IntegrationTestBase.java | 4 +++- .../apache/hadoop/hbase/IntegrationTestIngest.java | 17 +++++++---------- .../hbase/chaos/monkies/PolicyBasedChaosMonkey.java | 7 +++++++ .../hbase/test/IntegrationTestBigLinkedList.java | 3 +++ .../hbase/test/IntegrationTestLoadAndVerify.java | 6 ++++-- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java index 68db769..9cf1d57 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java @@ -70,8 +70,8 @@ public abstract class IntegrationTestBase extends AbstractHBaseTool { @Override protected int doWork() throws Exception { - setUpMonkey(); setUp(); + setUpMonkey(); int result = -1; try { runTestFromCommandLine(); @@ -86,10 +86,12 @@ public abstract class IntegrationTestBase extends AbstractHBaseTool { @Before public void setUpMonkey() throws Exception { util = getTestingUtil(getConf()); + IntegrationTestingUtility.setUseDistributedCluster(getConf()); MonkeyFactory fact = MonkeyFactory.getFactory(monkeyToUse); monkey = fact.setUtil(util) .setTableName(getTablename()) .setColumnFamilies(getColumnFamilies()).build(); + monkey.start(); } @After diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngest.java hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngest.java index be72544..037cbe0 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngest.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngest.java @@ -41,8 +41,6 @@ public class IntegrationTestIngest extends IntegrationTestBase { private static final int SERVER_COUNT = 4; // number of slaves for the smallest cluster private static final long DEFAULT_RUN_TIME = 20 * 60 * 1000; - protected static String tableName = null; - /** A soft limit on how long we should run */ private static final String RUN_TIME_KEY = "hbase.%s.runtime"; @@ -52,7 +50,6 @@ public class IntegrationTestIngest extends IntegrationTestBase { private LoadTestTool loadTool; protected void setUp(int numSlavesBase) throws Exception { - tableName = this.getClass().getSimpleName(); util = getTestingUtil(null); LOG.debug("Initializing/checking cluster has " + numSlavesBase + " servers"); util.initializeCluster(numSlavesBase); @@ -63,7 +60,7 @@ public class IntegrationTestIngest extends IntegrationTestBase { loadTool.setConf(util.getConfiguration()); // Initialize load test tool before we start breaking things; // LoadTestTool init, even when it is a no-op, is very fragile. - int ret = loadTool.run(new String[] { "-tn", tableName, "-init_only" }); + int ret = loadTool.run(new String[] { "-tn", getTablename(), "-init_only" }); Assert.assertEquals("Failed to initialize LoadTestTool", 0, ret); } @@ -92,7 +89,7 @@ public class IntegrationTestIngest extends IntegrationTestBase { @Override public String getTablename() { - return tableName; + return this.getClass().getSimpleName(); } @Override @@ -101,8 +98,8 @@ public class IntegrationTestIngest extends IntegrationTestBase { } private void deleteTableIfNecessary() throws IOException { - if (util.getHBaseAdmin().tableExists(tableName)) { - util.deleteTable(Bytes.toBytes(tableName)); + if (util.getHBaseAdmin().tableExists(getTablename())) { + util.deleteTable(Bytes.toBytes(getTablename())); } } @@ -122,7 +119,7 @@ public class IntegrationTestIngest extends IntegrationTestBase { ((runtime - (System.currentTimeMillis() - start))/60000) + " min"); int ret = loadTool.run(new String[] { - "-tn", tableName, + "-tn", getTablename(), "-write", String.format("%d:%d:%d", colsPerKey, recordSize, writeThreads), "-start_key", String.valueOf(startKey), "-num_keys", String.valueOf(numKeys), @@ -135,7 +132,7 @@ public class IntegrationTestIngest extends IntegrationTestBase { } ret = loadTool.run(new String[] { - "-tn", tableName, + "-tn", getTablename(), "-update", String.format("60:%d", writeThreads), "-start_key", String.valueOf(startKey), "-num_keys", String.valueOf(numKeys), @@ -148,7 +145,7 @@ public class IntegrationTestIngest extends IntegrationTestBase { } ret = loadTool.run(new String[] { - "-tn", tableName, + "-tn", getTablename(), "-read", "100:20", "-start_key", String.valueOf(startKey), "-num_keys", String.valueOf(numKeys), diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java index d4528ef..5a0416e 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java @@ -139,6 +139,10 @@ public class PolicyBasedChaosMonkey extends ChaosMonkey { @Override public void stop(String why) { + if (policies == null) { + return; + } + for (Policy policy : policies) { policy.stop(why); } @@ -155,6 +159,9 @@ public class PolicyBasedChaosMonkey extends ChaosMonkey { */ @Override public void waitForStop() throws InterruptedException { + if (monkeyThreads == null) { + return; + } for (Thread monkeyThread : monkeyThreads) { monkeyThread.join(); } diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java index 10b3e2d..097b167 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java @@ -57,6 +57,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; import org.apache.hadoop.hbase.mapreduce.TableMapper; import org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl; +import org.apache.hadoop.hbase.util.AbstractHBaseTool; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.NullWritable; @@ -494,6 +495,7 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false); TableMapReduceUtil.addDependencyJars(job); + TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); TableMapReduceUtil.initCredentials(job); boolean success = job.waitForCompletion(true); @@ -637,6 +639,7 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { TableMapReduceUtil.initTableMapperJob(getTableName(getConf()).getName(), scan, VerifyMapper.class, BytesWritable.class, BytesWritable.class, job); + TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false); diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java index dab964d..b29a350 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.NMapInputFormat; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; import org.apache.hadoop.hbase.mapreduce.TableMapper; +import org.apache.hadoop.hbase.util.AbstractHBaseTool; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.NullWritable; @@ -311,8 +312,8 @@ public class IntegrationTestLoadAndVerify extends IntegrationTestBase { FileOutputFormat.setOutputPath(job, outputDir); TableMapReduceUtil.addDependencyJars(job); - TableMapReduceUtil.addDependencyJars( - job.getConfiguration(), HTable.class, Lists.class); + + TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); TableMapReduceUtil.initCredentials(job); assertTrue(job.waitForCompletion(true)); } @@ -329,6 +330,7 @@ public class IntegrationTestLoadAndVerify extends IntegrationTestBase { TableMapReduceUtil.initTableMapperJob( htd.getTableName().getNameAsString(), scan, VerifyMapper.class, BytesWritable.class, BytesWritable.class, job); + TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING); TableMapReduceUtil.setScannerCaching(job, scannerCaching); -- 1.7.10.2 (Apple Git-33)