From ceb8e95fc23c0357ab2e7d50b4f3610ee18a0cd4 Mon Sep 17 00:00:00 2001 From: Matt Warhaftig Date: Sun, 21 Jun 2015 11:55:11 -0400 Subject: [PATCH] HBASE-13708 - Add range option to random read test in PE. --- .../apache/hadoop/hbase/PerformanceEvaluation.java | 38 +++++++++++++++++++--- .../hadoop/hbase/TestPerformanceEvaluation.java | 25 ++++++++++++++ 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index 569ef71..5d23a2f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -404,7 +404,7 @@ public class PerformanceEvaluation extends Configured implements Tool { @Override public RunResult call() throws Exception { TestOptions threadOpts = new TestOptions(opts); - if (threadOpts.startRow == 0) threadOpts.startRow = index * threadOpts.perClientRunRows; + if (threadOpts.startRow == 0) threadOpts.startRow = index * threadOpts.rowRangeSize; RunResult run = runOneClient(cmd, conf, con, threadOpts, new Status() { @Override public void setStatus(final String msg) throws IOException { @@ -527,7 +527,7 @@ public class PerformanceEvaluation extends Configured implements Tool { for (int j = 0; j < opts.numClientThreads; j++) { TestOptions next = new TestOptions(opts); next.startRow = (j * perClientRows) + (i * (perClientRows/10)); - next.perClientRunRows = perClientRows / 10; + next.perClientRunRows = opts.perClientRunRows / 10; String s = MAPPER.writeValueAsString(next); LOG.info("Client=" + j + ", maptask=" + i + ", input=" + s); int hash = h.hash(Bytes.toBytes(s)); @@ -613,6 +613,7 @@ public class PerformanceEvaluation extends Configured implements Tool { int cycles = 1; int columns = 1; int caching = 30; + int rowRangeSize = 0; boolean addColumns = true; public TestOptions() {} @@ -658,6 +659,7 @@ public class PerformanceEvaluation extends Configured implements Tool { this.addColumns = that.addColumns; this.columns = that.columns; this.caching = that.caching; + this.rowRangeSize = that.rowRangeSize; } public int getCaching() { @@ -947,6 +949,14 @@ public class PerformanceEvaluation extends Configured implements Tool { public void setAddColumns(boolean addColumns) { this.addColumns = addColumns; } + + public void setRowRangeSize(int rowRangeSize) { + this.rowRangeSize = rowRangeSize; + } + + public int getRowRangeSize() { + return rowRangeSize; + } } /* @@ -1350,7 +1360,7 @@ public class PerformanceEvaluation extends Configured implements Tool { if (opts.randomSleep > 0) { Thread.sleep(rd.nextInt(opts.randomSleep)); } - Get get = new Get(getRandomRow(this.rand, opts.totalRows)); + Get get = new Get(getRandomRow(this.rand, opts.rowRangeSize)); if (opts.addColumns) { get.addColumn(FAMILY_NAME, QUALIFIER_NAME); } else { @@ -1650,6 +1660,7 @@ public class PerformanceEvaluation extends Configured implements Tool { status.setStatus("Finished " + cmd + " in " + totalElapsedTime + "ms at offset " + opts.startRow + " for " + opts.perClientRunRows + " rows" + + (opts.cmdName.equals("randomRead") ? " from a range of " + opts.rowRangeSize+ " rows" : "") + " (" + calculateMbps((int)(opts.perClientRunRows * opts.sampleRate), totalElapsedTime, getAverageValueLength(opts), opts.columns) + ")"); @@ -1753,6 +1764,8 @@ public class PerformanceEvaluation extends Configured implements Tool { System.err.println(" randomSleep Do a random sleep before each get between 0 and entered value. Defaults: 0"); System.err.println(" columns Columns to write per row. Default: 1"); System.err.println(" caching Scan caching to use. Default: 30"); + System.err.println(" rowRangeSize When doing RandomRead, will only read rows within " + + "this range. Default: rows value"); System.err.println(); System.err.println(" Note: -D properties will be applied to the conf used. "); System.err.println(" For example: "); @@ -1995,6 +2008,14 @@ public class PerformanceEvaluation extends Configured implements Tool { continue; } + final String rowRangeSize = "--rowRangeSize="; + if (cmd.startsWith(rowRangeSize)) { + opts.rowRangeSize = Integer.parseInt(cmd.substring(rowRangeSize.length())); + if (opts.rowRangeSize <= 0) + throw new IllegalStateException("RowRangeSize must be > 0; i.e. 10"); + continue; + } + if (isCommandClass(cmd)) { opts.cmdName = cmd; opts.numClientThreads = Integer.parseInt(args.remove()); @@ -2025,9 +2046,18 @@ public class PerformanceEvaluation extends Configured implements Tool { opts.perClientRunRows = opts.totalRows / opts.numClientThreads; } else if (opts.perClientRunRows != DEFAULT_OPTS.perClientRunRows) { // number of rows specified - opts.totalRows = opts.perClientRunRows * opts.numClientThreads; + if(opts.cmdName == null || !opts.cmdName.equals("randomRead") || opts.rowRangeSize == 0){ + opts.rowRangeSize = opts.perClientRunRows; + opts.totalRows = opts.rowRangeSize * opts.numClientThreads; + } else { + opts.totalRows = opts.rowRangeSize * opts.numClientThreads * (opts.nomapred ? 1 : + TASKS_PER_CLIENT); + } opts.size = opts.totalRows / rowsPerGB; } + if(opts.cmdName == null || !opts.cmdName.equals("randomRead") || opts.rowRangeSize == 0){ + opts.rowRangeSize = opts.perClientRunRows; + } return opts; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java index e35fc08..8dd6457 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java @@ -98,9 +98,11 @@ public class TestPerformanceEvaluation { TestOptions opts = new PerformanceEvaluation.TestOptions(); opts = PerformanceEvaluation.calculateRowsAndSize(opts); int rows = opts.getPerClientRunRows(); + int rowRangeSize = opts.getRowRangeSize(); // Default row count final int defaultPerClientRunRows = 1024 * 1024; assertEquals(defaultPerClientRunRows, rows); + assertEquals(defaultPerClientRunRows, rowRangeSize); // If size is 2G, then twice the row count. opts.setSize(2.0f); opts = PerformanceEvaluation.calculateRowsAndSize(opts); @@ -113,6 +115,29 @@ public class TestPerformanceEvaluation { opts.valueRandom = true; opts = PerformanceEvaluation.calculateRowsAndSize(opts); assertEquals(defaultPerClientRunRows * 2, opts.getPerClientRunRows()); + assertEquals(defaultPerClientRunRows * 2, opts.getRowRangeSize()); + } + + @Test + public void testRowRangeCalculation() { + TestOptions opts = new PerformanceEvaluation.TestOptions(); + opts.setPerClientRunRows(10); + opts.setNomapred(false); + opts = PerformanceEvaluation.calculateRowsAndSize(opts); + // All tests except randomRead get rowRangeSize = perClientRunRows. + assertEquals(opts.getRowRangeSize(), opts.perClientRunRows); + assertEquals(10, opts.getTotalRows()); + + opts = new PerformanceEvaluation.TestOptions(); + opts.setPerClientRunRows(10); + opts.setNomapred(false); + opts.setCmdName("randomRead"); + opts.setRowRangeSize(20); + opts = PerformanceEvaluation.calculateRowsAndSize(opts); + // randomRead can have rowRangeSize != perClientRunRows. + assertEquals(10, opts.getPerClientRunRows()); + assertEquals(20, opts.getRowRangeSize()); + assertEquals(200, opts.getTotalRows()); } @Test -- 2.3.2 (Apple Git-55)