diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java index cb9f311..d41800d 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java @@ -32,6 +32,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.math.stat.descriptive.DescriptiveStatistics; import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.IntegrationTestingUtility; @@ -51,7 +52,6 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.KeyOnlyFilter; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.LoadTestTool; -import org.cloudera.htrace.Sampler; import org.cloudera.htrace.Span; import org.cloudera.htrace.Trace; import org.cloudera.htrace.TraceScope; @@ -61,7 +61,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; -import static junit.framework.Assert.assertEquals; +import static org.junit.Assert.assertEquals; /** * Integration test that should benchmark how fast HBase can recover from failures. This test starts @@ -354,28 +354,40 @@ public class IntegrationTestMTTR { * Base class for actions that need to record the time needed to recover from a failure. */ public abstract class TimingCallable implements Callable { - protected final Future future; + protected final Future future; - public TimingCallable(Future f) { + public TimingCallable(Future f) { future = f; } @Override public TimingResult call() throws Exception { TimingResult result = new TimingResult(); + final int maxIterations = 10; int numAfterDone = 0; + int resetCount = 0; // Keep trying until the rs is back up and we've gotten a put through - while (numAfterDone < 10) { + while (numAfterDone < maxIterations) { long start = System.nanoTime(); TraceScope scope = null; try { scope = Trace.startSpan(getSpanName(), AlwaysSampler.INSTANCE); boolean actionResult = doAction(); if (actionResult && future.isDone()) { - numAfterDone ++; + numAfterDone++; } + } catch (DoNotRetryIOException e) { + // something is critically broken, abort test. + throw e; } catch (Exception e) { - numAfterDone = 0; + resetCount++; + if (resetCount < maxIterations) { + LOG.info("Exception while running " + this.toString() + ". Resetting loop counter", e); + numAfterDone = 0; + } else { + LOG.info("Too many unexpected Exceptions. Aborting.", e); + throw e; + } } finally { if (scope != null) { scope.close(); @@ -391,6 +403,11 @@ public class IntegrationTestMTTR { protected String getSpanName() { return this.getClass().getSimpleName(); } + + @Override + public String toString() { + return this.getSpanName(); + } } /** @@ -401,7 +418,7 @@ public class IntegrationTestMTTR { private final HTable table; - public PutCallable(Future f) throws IOException { + public PutCallable(Future f) throws IOException { super(f); this.table = new HTable(util.getConfiguration(), tableName); } @@ -428,7 +445,7 @@ public class IntegrationTestMTTR { public class ScanCallable extends TimingCallable { private final HTable table; - public ScanCallable(Future f) throws IOException { + public ScanCallable(Future f) throws IOException { super(f); this.table = new HTable(util.getConfiguration(), tableName); } @@ -463,15 +480,20 @@ public class IntegrationTestMTTR { */ public class AdminCallable extends TimingCallable { - public AdminCallable(Future f) throws IOException { + public AdminCallable(Future f) throws IOException { super(f); } @Override protected boolean doAction() throws Exception { - HBaseAdmin admin = new HBaseAdmin(util.getConfiguration()); - ClusterStatus status = admin.getClusterStatus(); - return status != null; + HBaseAdmin admin = null; + try { + admin = new HBaseAdmin(util.getConfiguration()); + ClusterStatus status = admin.getClusterStatus(); + return status != null; + } finally { + if (null != admin) admin.close(); + } } @Override @@ -501,9 +523,9 @@ public class IntegrationTestMTTR { */ public class LoadCallable implements Callable { - private final Future future; + private final Future future; - public LoadCallable(Future f) { + public LoadCallable(Future f) { future = f; } @@ -530,4 +552,4 @@ public class IntegrationTestMTTR { return true; } } -} \ No newline at end of file +}