commit 30b83d79e5d776a20959d3d656fd1a350c60c073 Author: stack Date: Fri Feb 5 11:18:42 2016 -0800 HBASE-15157 Add *PerformanceTest for Append, CheckAnd* diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index 30a7293..e4c043a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -52,19 +52,24 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.BufferedMutator; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Consistency; +import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Increment; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.filter.BinaryComparator; import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterAllFilter; import org.apache.hadoop.hbase.filter.FilterList; @@ -167,7 +172,17 @@ public class PerformanceEvaluation extends Configured implements Tool { "Run scan test (read every row)"); addCommandDescriptor(FilteredScanTest.class, "filterScan", "Run scan test using a filter to find a specific row based on it's value " + - "(make sure to use --rows=20)"); + "(make sure to use --rows=20)"); + addCommandDescriptor(IncrementTest.class, "increment", + "Increment on each row; clients overlap on keyspace so some concurrent operations"); + addCommandDescriptor(AppendTest.class, "append", + "Append on each row; clients overlap on keyspace so some concurrent operations"); + addCommandDescriptor(CheckAndMutateTest.class, "checkAndMutate", + "CheckAndMutate on each row; clients overlap on keyspace so some concurrent operations"); + addCommandDescriptor(CheckAndPutTest.class, "checkAndPut", + "CheckAndPut on each row; clients overlap on keyspace so some concurrent operations"); + addCommandDescriptor(CheckAndDeleteTest.class, "checkAndDelete", + "CheckAndDelete on each row; clients overlap on keyspace so some concurrent operations"); } /** @@ -1057,13 +1072,22 @@ public class PerformanceEvaluation extends Configured implements Tool { return (System.nanoTime() - startTime) / 1000000; } + int getStartRow() { + return opts.startRow; + } + + int getLastRow() { + return getStartRow() + opts.perClientRunRows; + } + /** * Provides an extension point for tests that don't want a per row invocation. */ void testTimed() throws IOException, InterruptedException { - int lastRow = opts.startRow + opts.perClientRunRows; + int startRow = getStartRow(); + int lastRow = getLastRow(); // Report on completion of 1/10th of total. - for (int i = opts.startRow; i < lastRow; i++) { + for (int i = startRow; i < lastRow; i++) { if (i % everyN != 0) continue; long startTime = System.nanoTime(); TraceScope scope = Trace.startSpan("test row", traceSampler); @@ -1074,7 +1098,7 @@ public class PerformanceEvaluation extends Configured implements Tool { } latency.update((System.nanoTime() - startTime) / 1000); if (status != null && i > 0 && (i % getReportingPeriod()) == 0) { - status.setStatus(generateStatus(opts.startRow, i, lastRow)); + status.setStatus(generateStatus(startRow, i, lastRow)); } } } @@ -1419,7 +1443,116 @@ public class PerformanceEvaluation extends Configured implements Tool { Result r = testScanner.next(); updateValueSize(r); } + } + + /** + * Base class for operations that are CAS-like; that read a value and then set it based off what + * they read. In this category is increment, append, checkAndPut, etc. + * + *

These operations also want some concurrency going on. Usually when these tests run, they + * operate in their own part of the key range. In CASTest, we will have them all overlap on the + * same key space. We do this with our getStartRow and getLastRow overrides. + */ + static abstract class CASTableTest extends TableTest { + private final byte [] qualifier; + CASTableTest(Connection con, TestOptions options, Status status) { + super(con, options, status); + qualifier = Bytes.toBytes(this.getClass().getSimpleName()); + } + + byte [] getQualifier() { + return this.qualifier; + } + + @Override + int getStartRow() { + return 0; + } + @Override + int getLastRow() { + return opts.perClientRunRows; + } + } + + static class IncrementTest extends CASTableTest { + IncrementTest(Connection con, TestOptions options, Status status) { + super(con, options, status); + } + + @Override + void testRow(final int i) throws IOException { + Increment increment = new Increment(format(i)); + increment.addColumn(FAMILY_NAME, getQualifier(), 1l); + updateValueSize(this.table.increment(increment)); + } + } + + static class AppendTest extends CASTableTest { + AppendTest(Connection con, TestOptions options, Status status) { + super(con, options, status); + } + + @Override + void testRow(final int i) throws IOException { + byte [] bytes = format(i); + Append append = new Append(bytes); + append.add(FAMILY_NAME, getQualifier(), bytes); + updateValueSize(this.table.append(append)); + } + } + + static class CheckAndMutateTest extends CASTableTest { + CheckAndMutateTest(Connection con, TestOptions options, Status status) { + super(con, options, status); + } + + @Override + void testRow(final int i) throws IOException { + byte [] bytes = format(i); + // Put a known value so when we go to check it, it is there. + Put put = new Put(bytes); + put.addColumn(FAMILY_NAME, getQualifier(), bytes); + this.table.put(put); + RowMutations mutations = new RowMutations(bytes); + mutations.add(put); + this.table.checkAndMutate(bytes, FAMILY_NAME, getQualifier(), CompareOp.EQUAL, bytes, + mutations); + } + } + + static class CheckAndPutTest extends CASTableTest { + CheckAndPutTest(Connection con, TestOptions options, Status status) { + super(con, options, status); + } + + @Override + void testRow(final int i) throws IOException { + byte [] bytes = format(i); + // Put a known value so when we go to check it, it is there. + Put put = new Put(bytes); + put.addColumn(FAMILY_NAME, getQualifier(), bytes); + this.table.put(put); + this.table.checkAndPut(bytes, FAMILY_NAME, getQualifier(), CompareOp.EQUAL, bytes, put); + } + } + + static class CheckAndDeleteTest extends CASTableTest { + CheckAndDeleteTest(Connection con, TestOptions options, Status status) { + super(con, options, status); + } + + @Override + void testRow(final int i) throws IOException { + byte [] bytes = format(i); + // Put a known value so when we go to check it, it is there. + Put put = new Put(bytes); + put.addColumn(FAMILY_NAME, getQualifier(), bytes); + this.table.put(put); + Delete delete = new Delete(put.getRow()); + delete.addColumn(FAMILY_NAME, getQualifier()); + this.table.checkAndDelete(bytes, FAMILY_NAME, getQualifier(), CompareOp.EQUAL, bytes, delete); + } } static class SequentialReadTest extends TableTest { @@ -1723,8 +1856,10 @@ public class PerformanceEvaluation extends Configured implements Tool { "clients (and HRegionServers)"); System.err.println(" running: 1 <= value <= 500"); System.err.println("Examples:"); - System.err.println(" To run a single evaluation client:"); + System.err.println(" To run a single client doing the default 1M sequentialWrites:"); System.err.println(" $ bin/hbase " + className + " sequentialWrite 1"); + System.err.println(" To run 10 clients doing increments over ten rows:"); + System.err.println(" $ bin/hbase " + className + " --rows=10 --nomapred increment 10"); } /**