Index: hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java (revision 1568198) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java (working copy) @@ -137,6 +137,7 @@ private int presplitRegions = 0; private boolean useTags = false; private int noOfTags = 1; + private int multiGet = 0; private HConnection connection; private static final Path PERF_EVAL_DIR = new Path("performance_evaluation"); @@ -153,7 +154,8 @@ "writeToWAL=(\\w+),\\s+" + "useTags=(\\w+),\\s+" + "noOfTags=(\\d+),\\s+" + - "reportLatency=(\\w+)"); + "reportLatency=(\\w+),\\s+" + + "multiGet=(\\d+)"); /** * Enum for map metrics. Keep it out here rather than inside in the Map @@ -235,12 +237,13 @@ private boolean useTags = false; private int noOfTags = 0; private boolean reportLatency = false; + private int multiGet = 0; public PeInputSplit() {} public PeInputSplit(TableName tableName, int startRow, int rows, int totalRows, float sampleRate, int clients, boolean flushCommits, boolean writeToWAL, - boolean useTags, int noOfTags, boolean reportLatency) { + boolean useTags, int noOfTags, boolean reportLatency, int multiGet) { this.tableName = tableName; this.startRow = startRow; this.rows = rows; @@ -252,6 +255,7 @@ this.useTags = useTags; this.noOfTags = noOfTags; this.reportLatency = reportLatency; + this.multiGet = multiGet; } @Override @@ -271,6 +275,7 @@ this.useTags = in.readBoolean(); this.noOfTags = in.readInt(); this.reportLatency = in.readBoolean(); + this.multiGet = in.readInt(); } @Override @@ -288,6 +293,7 @@ out.writeBoolean(useTags); out.writeInt(noOfTags); out.writeBoolean(reportLatency); + out.writeInt(multiGet); } @Override @@ -343,6 +349,10 @@ public boolean isReportLatency() { return reportLatency; } + + public int getMultiGet() { + return multiGet; + } } /** @@ -384,6 +394,7 @@ boolean useTags = Boolean.parseBoolean(m.group(9)); int noOfTags = Integer.parseInt(m.group(10)); boolean reportLatency = Boolean.parseBoolean(m.group(11)); + int multiGet = Integer.parseInt(m.group(12)); LOG.debug("tableName=" + tableName + " split["+ splitList.size() + "] " + @@ -396,11 +407,12 @@ " writeToWAL=" + writeToWAL + " useTags=" + useTags + " noOfTags=" + noOfTags + - " reportLatency=" + reportLatency); + " reportLatency=" + reportLatency + + " multiGet=" + multiGet); PeInputSplit newSplit = new PeInputSplit(tableName, startRow, rows, totalRows, sampleRate, clients, - flushCommits, writeToWAL, useTags, noOfTags, reportLatency); + flushCommits, writeToWAL, useTags, noOfTags, reportLatency, multiGet); splitList.add(newSplit); } } @@ -523,7 +535,7 @@ long elapsedTime = this.pe.runOneClient(this.cmd, value.getStartRow(), value.getRows(), value.getTotalRows(), value.getSampleRate(), value.isFlushCommits(), value.isWriteToWAL(), value.isUseTags(), - value.getNoOfTags(), value.isReportLatency(), + value.getNoOfTags(), value.isReportLatency(), value.getMultiGet(), HConnectionManager.createConnection(context.getConfiguration()), status); // Collect how much time the thing took. Report as map output and // to the ELAPSED_TIME counter. @@ -657,7 +669,7 @@ try { long elapsedTime = pe.runOneClient(cmd, index * perClientRows, perClientRows, R, sampleRate, flushCommits, writeToWal, useTags, - noOfTags, reportLatency, connection, new Status() { + noOfTags, reportLatency, multiGet, connection, new Status() { public void setStatus(final String msg) throws IOException { LOG.info("client-" + getName() + " " + msg); } @@ -829,13 +841,15 @@ private boolean useTags = false; private int noOfTags = 0; private boolean reportLatency; + private int multiGet = 0; private HConnection connection; TestOptions() {} TestOptions(int startRow, int perClientRunRows, int totalRows, float sampleRate, int numClientThreads, TableName tableName, boolean flushCommits, boolean writeToWAL, - boolean useTags, int noOfTags, boolean reportLatency, HConnection connection) { + boolean useTags, int noOfTags, boolean reportLatency, int multiGet, + HConnection connection) { this.startRow = startRow; this.perClientRunRows = perClientRunRows; this.totalRows = totalRows; @@ -847,6 +861,7 @@ this.useTags = useTags; this.noOfTags = noOfTags; this.reportLatency = reportLatency; + this.multiGet = multiGet; this.connection = connection; } @@ -886,6 +901,10 @@ return reportLatency; } + public int getMultiGet() { + return multiGet; + } + public HConnection getConnection() { return connection; } @@ -1121,7 +1140,9 @@ static class RandomReadTest extends Test { private final int everyN; private final boolean reportLatency; - private final double[] times; + private final int multiGet; + private float[] times; + private ArrayList gets; int idx = 0; RandomReadTest(Configuration conf, TestOptions options, Status status) { @@ -1129,10 +1150,12 @@ everyN = (int) (this.totalRows / (this.totalRows * this.sampleRate)); LOG.info("Sampling 1 every " + everyN + " out of " + perClientRunRows + " total rows."); this.reportLatency = options.isReportLatency(); + this.multiGet = options.getMultiGet(); + if (this.multiGet > 0) { + this.gets = new ArrayList(this.multiGet); + } if (this.reportLatency) { - times = new double[(int) Math.ceil(this.perClientRunRows * this.sampleRate)]; - } else { - times = null; + this.times = new float[(int) Math.ceil(this.perClientRunRows * this.sampleRate / Math.max(1, this.multiGet))]; } } @@ -1141,10 +1164,22 @@ if (i % everyN == 0) { Get get = new Get(getRandomRow(this.rand, this.totalRows)); get.addColumn(FAMILY_NAME, QUALIFIER_NAME); - long start = System.nanoTime(); - this.table.get(get); - if (this.reportLatency) { - times[idx++] = (float) ((System.nanoTime() - start) / 1000000.0); + if (this.multiGet > 0) { + this.gets.add(get); + if (this.gets.size() == this.multiGet) { + long start = System.nanoTime(); + this.table.get(this.gets); + if (this.reportLatency) { + times[idx++] = (float) ((System.nanoTime() - start) / 1000000.0); + } + this.gets.clear(); + } + } else { + long start = System.nanoTime(); + this.table.get(get); + if (this.reportLatency) { + times[idx++] = (float) ((System.nanoTime() - start) / 1000000.0); + } } } } @@ -1157,6 +1192,10 @@ @Override protected void testTakedown() throws IOException { + if (this.gets != null && this.gets.size() > 0) { + this.table.get(gets); + this.gets.clear(); + } super.testTakedown(); if (this.reportLatency) { DescriptiveStatistics ds; @@ -1377,7 +1416,7 @@ long runOneClient(final Class cmd, final int startRow, final int perClientRunRows, final int totalRows, final float sampleRate, boolean flushCommits, boolean writeToWAL, boolean useTags, int noOfTags, - boolean reportLatency, HConnection connection, final Status status) + boolean reportLatency, int multiGet, HConnection connection, final Status status) throws IOException { status.setStatus("Start " + cmd + " at offset " + startRow + " for " + perClientRunRows + " rows"); @@ -1385,7 +1424,7 @@ TestOptions options = new TestOptions(startRow, perClientRunRows, totalRows, sampleRate, N, tableName, flushCommits, writeToWAL, useTags, noOfTags, - reportLatency, connection); + reportLatency, multiGet, connection); final Test t; try { Constructor constructor = cmd.getDeclaredConstructor( @@ -1419,7 +1458,8 @@ admin = new HBaseAdmin(getConf()); checkTable(admin); runOneClient(cmd, 0, this.R, this.R, this.sampleRate, this.flushCommits, - this.writeToWAL, this.useTags, this.noOfTags, this.reportLatency, this.connection, status); + this.writeToWAL, this.useTags, this.noOfTags, this.reportLatency, this.multiGet, + this.connection, status); } catch (Exception e) { LOG.error("Failed", e); } finally { @@ -1597,6 +1637,12 @@ continue; } + final String multiGet = "--multiGet="; + if (cmd.startsWith(multiGet)) { + this.multiGet = Integer.parseInt(cmd.substring(multiGet.length())); + continue; + } + this.connection = HConnectionManager.createConnection(getConf()); final String useTags = "--usetags=";