From c849dd44fe0828d3a17b05e1d796bb5752ae1934 Mon Sep 17 00:00:00 2001 From: Elliott Clark Date: Tue, 9 Jul 2013 11:08:36 -0700 Subject: [PATCH] HBASE-8904 clean up mttr it test --- .../hadoop/hbase/mttr/IntegrationTestMTTR.java | 123 +++++++++++++++----- 1 file changed, 96 insertions(+), 27 deletions(-) diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java hbase-it/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java index 256f9bf..3940835 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java @@ -22,6 +22,7 @@ import com.google.common.base.Objects; import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.commons.math.stat.descriptive.DescriptiveStatistics; import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; @@ -37,6 +38,9 @@ import org.apache.hadoop.hbase.filter.KeyOnlyFilter; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChaosMonkey; import org.apache.hadoop.hbase.util.LoadTestTool; +import org.cloudera.htrace.Sampler; +import org.cloudera.htrace.Span; +import org.cloudera.htrace.Trace; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -106,6 +110,7 @@ public class IntegrationTestMTTR { private static String tableName; private static byte[] tableNameBytes; private static String loadTableName; + private static byte[] loadTableNameBytes; /** * Util to get at the cluster. @@ -130,6 +135,7 @@ public class IntegrationTestMTTR { */ private static LoadTestTool loadTool; + @BeforeClass public static void setUp() throws Exception { // Set up the integration test util @@ -185,20 +191,21 @@ public class IntegrationTestMTTR { loadTableName = util.getConfiguration() .get("hbase.IntegrationTestMTTR.loadTableName", "IntegrationTestMTTRLoadTestTool"); + loadTableNameBytes = Bytes.toBytes(loadTableName); if (util.getHBaseAdmin().tableExists(tableNameBytes)) { util.deleteTable(tableNameBytes); } - if (util.getHBaseAdmin().tableExists(loadTableName)) { - util.deleteTable(loadTableName); + if (util.getHBaseAdmin().tableExists(loadTableNameBytes)) { + util.deleteTable(loadTableNameBytes); } // Create the table. If this fails then fail everything. HTableDescriptor tableDescriptor = new HTableDescriptor(tableNameBytes); // Make the max file size huge so that splits don't happen during the test. - tableDescriptor.setMaxFileSize(2 * 1024 * 1024 * 1024); + tableDescriptor.setMaxFileSize(Long.MAX_VALUE); HColumnDescriptor descriptor = new HColumnDescriptor(FAMILY); descriptor.setMaxVersions(1); @@ -253,9 +260,9 @@ public class IntegrationTestMTTR { int maxIters = util.getHBaseClusterInterface().isDistributedCluster() ? 10 : 3; // Array to keep track of times. - ArrayList timesPutMs = new ArrayList(maxIters); - ArrayList timesScanMs = new ArrayList(maxIters); - ArrayList timesAdminMs = new ArrayList(maxIters); + ArrayList resultPuts = new ArrayList(maxIters); + ArrayList resultScan = new ArrayList(maxIters); + ArrayList resultAdmin = new ArrayList(maxIters); long start = System.nanoTime(); // We're going to try this multiple times @@ -264,9 +271,9 @@ public class IntegrationTestMTTR { Future monkeyFuture = executorService.submit(monkeyCallable); // Pass that future to the timing Callables. - Future putFuture = executorService.submit(new PutCallable(monkeyFuture)); - Future scanFuture = executorService.submit(new ScanCallable(monkeyFuture)); - Future adminFuture = executorService.submit(new AdminCallable(monkeyFuture)); + Future putFuture = executorService.submit(new PutCallable(monkeyFuture)); + Future scanFuture = executorService.submit(new ScanCallable(monkeyFuture)); + Future adminFuture = executorService.submit(new AdminCallable(monkeyFuture)); Future loadFuture = executorService.submit(new LoadCallable(monkeyFuture)); @@ -274,14 +281,14 @@ public class IntegrationTestMTTR { loadFuture.get(); // Get the values from the futures. - long putTime = TimeUnit.MILLISECONDS.convert(putFuture.get(), TimeUnit.NANOSECONDS); - long scanTime = TimeUnit.MILLISECONDS.convert(scanFuture.get(), TimeUnit.NANOSECONDS); - long adminTime = TimeUnit.MILLISECONDS.convert(adminFuture.get(), TimeUnit.NANOSECONDS); + TimingResult putTime = putFuture.get(); + TimingResult scanTime = scanFuture.get(); + TimingResult adminTime = adminFuture.get(); // Store the times to display later. - timesPutMs.add(putTime); - timesScanMs.add(scanTime); - timesAdminMs.add(adminTime); + resultPuts.add(putTime); + resultScan.add(scanTime); + resultAdmin.add(adminTime); // Wait some time for everything to settle down. Thread.sleep(5000l); @@ -290,9 +297,9 @@ public class IntegrationTestMTTR { long runtimeMs = TimeUnit.MILLISECONDS.convert(System.nanoTime() - start, TimeUnit.NANOSECONDS); Objects.ToStringHelper helper = Objects.toStringHelper("MTTRResults") - .add("timesPutMs", timesPutMs) - .add("timesScanMs", timesScanMs) - .add("timesAdminMs", timesAdminMs) + .add("putResults", resultPuts) + .add("scanResults", resultScan) + .add("adminResults", resultAdmin) .add("totalRuntimeMs", runtimeMs) .add("name", testName); @@ -301,9 +308,49 @@ public class IntegrationTestMTTR { } /** + * Class to store results of TimingCallable. + * + * Stores times and trace id. + */ + private class TimingResult { + DescriptiveStatistics stats = new DescriptiveStatistics(); + ArrayList traces = new ArrayList(10); + + /** + * Add a result to this aggregate result. + * @param time Time in nanoseconds + * @param span Span. To be kept if the time taken was over 1 second + */ + public void addResult(long time, Span span) { + stats.addValue(TimeUnit.MILLISECONDS.convert(time, TimeUnit.NANOSECONDS)); + if (TimeUnit.SECONDS.convert(time, TimeUnit.NANOSECONDS) >= 1) { + traces.add(span.getTraceId()); + } + } + + public String toString() { + Objects.ToStringHelper helper = Objects.toStringHelper(this) + .add("numResults", stats.getN()) + .add("minTime", stats.getMin()) + .add("meanTime", stats.getMean()) + .add("maxTime", stats.getMax()) + .add("25th", stats.getPercentile(25)) + .add("50th", stats.getPercentile(50)) + .add("75th", stats.getPercentile(75)) + .add("90th", stats.getPercentile(90)) + .add("95th", stats.getPercentile(95)) + .add("99th", stats.getPercentile(99)) + .add("99.9th", stats.getPercentile(99.9)) + .add("99.99th", stats.getPercentile(99.99)) + .add("traces", traces); + return helper.toString(); + } + } + + /** * Base class for actions that need to record the time needed to recover from a failure. */ - public abstract class TimingCallable implements Callable { + public abstract class TimingCallable implements Callable { protected final Future future; public TimingCallable(Future f) { @@ -311,27 +358,36 @@ public class IntegrationTestMTTR { } @Override - public Long call() throws Exception { - // TODO(eclark): Do more than just max. Full histogram support. - long maxTime = 0; + public TimingResult call() throws Exception { + TimingResult result = new TimingResult(); int numAfterDone = 0; // Keep trying until the rs is back up and we've gotten a put through while (numAfterDone < 10) { long start = System.nanoTime(); + Span span = null; try { - boolean result = doAction(); - if (result && future.isDone()) { + span = Trace.startSpan(getSpanName(), Sampler.ALWAYS); + boolean actionResult = doAction(); + if (actionResult && future.isDone()) { numAfterDone ++; } } catch (Exception e) { numAfterDone = 0; + } finally { + if (span != null) { + span.stop(); + } } - maxTime = Math.max(maxTime, System.nanoTime() - start); + result.addResult(System.nanoTime() - start, span); } - return maxTime; + return result; } protected abstract boolean doAction() throws Exception; + + protected String getSpanName() { + return this.getClass().getSimpleName(); + } } /** @@ -355,6 +411,11 @@ public class IntegrationTestMTTR { table.flushCommits(); return true; } + + @Override + protected String getSpanName() { + return "MTTR Put Test"; + } } /** @@ -388,6 +449,10 @@ public class IntegrationTestMTTR { } } } + @Override + protected String getSpanName() { + return "MTTR Scan Test"; + } } /** @@ -405,6 +470,11 @@ public class IntegrationTestMTTR { ClusterStatus status = admin.getClusterStatus(); return status != null; } + + @Override + protected String getSpanName() { + return "MTTR Admin Test"; + } } @@ -412,7 +482,6 @@ public class IntegrationTestMTTR { private final ChaosMonkey.Action action; public ActionCallable(ChaosMonkey.Action action) { - this.action = action; } -- 1.7.10.2 (Apple Git-33)