diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index f973f03..54b27e5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -133,6 +133,7 @@ public class PerformanceEvaluation extends Configured implements Tool { private boolean inMemoryCF = false; private boolean reportLatency = false; private int presplitRegions = 0; + private int multiGet = 0; private HConnection connection; private static final Path PERF_EVAL_DIR = new Path("performance_evaluation"); @@ -147,7 +148,8 @@ public class PerformanceEvaluation extends Configured implements Tool { "clients=(\\d+),\\s+" + "flushCommits=(\\w+),\\s+" + "writeToWAL=(\\w+),\\s+" + - "reportLatency=(\\w+)"); + "reportLatency=(\\w+),\\s+" + + "multiGet=(\\d+)"); /** * Enum for map metrics. Keep it out here rather than inside in the Map @@ -226,12 +228,13 @@ public class PerformanceEvaluation extends Configured implements Tool { private boolean flushCommits = false; private boolean writeToWAL = true; private boolean reportLatency = false; + private int multiGet = 0; public PeInputSplit() {} public PeInputSplit(TableName tableName, int startRow, int rows, int totalRows, float sampleRate, int clients, boolean flushCommits, boolean writeToWAL, - boolean reportLatency) { + boolean reportLatency, int multiGet) { this.tableName = tableName; this.startRow = startRow; this.rows = rows; @@ -241,6 +244,7 @@ public class PerformanceEvaluation extends Configured implements Tool { this.flushCommits = flushCommits; this.writeToWAL = writeToWAL; this.reportLatency = reportLatency; + this.multiGet = multiGet; } @Override @@ -258,6 +262,7 @@ public class PerformanceEvaluation extends Configured implements Tool { this.flushCommits = in.readBoolean(); this.writeToWAL = in.readBoolean(); this.reportLatency = in.readBoolean(); + this.multiGet = in.readInt(); } @Override @@ -273,6 +278,7 @@ public class PerformanceEvaluation extends Configured implements Tool { out.writeBoolean(flushCommits); out.writeBoolean(writeToWAL); out.writeBoolean(reportLatency); + out.writeInt(multiGet); } @Override @@ -320,6 +326,10 @@ public class PerformanceEvaluation extends Configured implements Tool { public boolean isReportLatency() { return reportLatency; } + + public int getMultiGet() { + return multiGet; + } } /** @@ -359,6 +369,7 @@ public class PerformanceEvaluation extends Configured implements Tool { boolean flushCommits = Boolean.parseBoolean(m.group(7)); boolean writeToWAL = Boolean.parseBoolean(m.group(8)); boolean reportLatency = Boolean.parseBoolean(m.group(9)); + int multiGet = Integer.parseInt(m.group(10)); LOG.debug("tableName=" + tableName + " split["+ splitList.size() + "] " + @@ -369,11 +380,12 @@ public class PerformanceEvaluation extends Configured implements Tool { " clients=" + clients + " flushCommits=" + flushCommits + " writeToWAL=" + writeToWAL + - " reportLatency=" + reportLatency); + " reportLatency=" + reportLatency + + " multiGet=" + multiGet); PeInputSplit newSplit = new PeInputSplit(tableName, startRow, rows, totalRows, sampleRate, clients, - flushCommits, writeToWAL, reportLatency); + flushCommits, writeToWAL, reportLatency, multiGet); splitList.add(newSplit); } } @@ -496,7 +508,8 @@ public class PerformanceEvaluation extends Configured implements Tool { long elapsedTime = this.pe.runOneClient(this.cmd, value.getStartRow(), value.getRows(), value.getTotalRows(), value.getSampleRate(), value.isFlushCommits(), value.isWriteToWAL(), value.isReportLatency(), - HConnectionManager.createConnection(context.getConfiguration()), status); + value.getMultiGet(), HConnectionManager.createConnection(context.getConfiguration()), + status); // Collect how much time the thing took. Report as map output and // to the ELAPSED_TIME counter. context.getCounter(Counter.ELAPSED_TIME).increment(elapsedTime); @@ -604,6 +617,7 @@ public class PerformanceEvaluation extends Configured implements Tool { final boolean writeToWal = this.writeToWAL; final boolean reportLatency = this.reportLatency; final int preSplitRegions = this.presplitRegions; + final int multiGet = this.multiGet; final HConnection connection = HConnectionManager.createConnection(getConf()); for (int i = 0; i < this.N; i++) { final int index = i; @@ -622,10 +636,11 @@ public class PerformanceEvaluation extends Configured implements Tool { pe.sampleRate = sampleRate; pe.reportLatency = reportLatency; pe.connection = connection; + pe.multiGet = multiGet; try { long elapsedTime = pe.runOneClient(cmd, index * perClientRows, - perClientRows, R, sampleRate, - flushCommits, writeToWAL, reportLatency, connection, new Status() { + perClientRows, R, sampleRate, flushCommits, writeToWAL, reportLatency, multiGet, + connection, new Status() { public void setStatus(final String msg) throws IOException { LOG.info("client-" + getName() + " " + msg); } @@ -738,7 +753,8 @@ public class PerformanceEvaluation extends Configured implements Tool { ", clients=" + this.N + ", flushCommits=" + this.flushCommits + ", writeToWAL=" + this.writeToWAL + - ", reportLatency=" + this.reportLatency; + ", reportLatency=" + this.reportLatency + + ", multiGet=" + this.multiGet; int hash = h.hash(Bytes.toBytes(s)); m.put(hash, s); } @@ -793,11 +809,12 @@ public class PerformanceEvaluation extends Configured implements Tool { private boolean flushCommits; private boolean writeToWAL = true; private boolean reportLatency; + private int multiGet = 0; private HConnection connection; TestOptions(int startRow, int perClientRunRows, int totalRows, float sampleRate, int numClientThreads, TableName tableName, boolean flushCommits, boolean writeToWAL, - boolean reportLatency, HConnection connection) { + boolean reportLatency, int multiGet, HConnection connection) { this.startRow = startRow; this.perClientRunRows = perClientRunRows; this.totalRows = totalRows; @@ -807,6 +824,7 @@ public class PerformanceEvaluation extends Configured implements Tool { this.flushCommits = flushCommits; this.writeToWAL = writeToWAL; this.reportLatency = reportLatency; + this.multiGet = multiGet; this.connection = connection; } @@ -846,6 +864,10 @@ public class PerformanceEvaluation extends Configured implements Tool { return reportLatency; } + public int getMultiGet() { + return multiGet; + } + public HConnection getConnection() { return connection; } @@ -1071,6 +1093,8 @@ public class PerformanceEvaluation extends Configured implements Tool { private final int everyN; private final boolean reportLatency; private final double[] times; + private final int multiGet; + private ArrayList gets; int idx = 0; RandomReadTest(Configuration conf, TestOptions options, Status status) { @@ -1078,10 +1102,15 @@ public class PerformanceEvaluation extends Configured implements Tool { everyN = (int) (this.totalRows / (this.totalRows * this.sampleRate)); LOG.info("Sampling 1 every " + everyN + " out of " + perClientRunRows + " total rows."); this.reportLatency = options.isReportLatency(); + this.multiGet = options.getMultiGet(); + if (this.multiGet > 0) { + LOG.info("MultiGet enabled. Sending GETs in batches of " + this.multiGet + "."); + this.gets = new ArrayList(this.multiGet); + } if (this.reportLatency) { - times = new double[(int) Math.ceil(this.perClientRunRows * this.sampleRate)]; + this.times = new double[(int) Math.ceil(this.perClientRunRows * this.sampleRate / Math.max(1, this.multiGet))]; } else { - times = null; + this.times = null; } } @@ -1090,10 +1119,22 @@ public class PerformanceEvaluation extends Configured implements Tool { if (i % everyN == 0) { Get get = new Get(getRandomRow(this.rand, this.totalRows)); get.addColumn(FAMILY_NAME, QUALIFIER_NAME); - long start = System.nanoTime(); - this.table.get(get); - if (this.reportLatency) { - times[idx++] = (float) ((System.nanoTime() - start) / 1000000.0); + if (this.multiGet > 0) { + this.gets.add(get); + if (this.gets.size() == this.multiGet) { + long start = System.nanoTime(); + this.table.get(this.gets); + if (this.reportLatency) { + times[idx++] = (System.nanoTime() - start) / 1e6; + } + this.gets.clear(); + } + } else { + long start = System.nanoTime(); + this.table.get(get); + if (this.reportLatency) { + times[idx++] = (System.nanoTime() - start) / 1e6; + } } } } @@ -1106,6 +1147,10 @@ public class PerformanceEvaluation extends Configured implements Tool { @Override protected void testTakedown() throws IOException { + if (this.gets != null && this.gets.size() > 0) { + this.table.get(gets); + this.gets.clear(); + } super.testTakedown(); if (this.reportLatency) { Arrays.sort(times); @@ -1302,7 +1347,7 @@ public class PerformanceEvaluation extends Configured implements Tool { long runOneClient(final Class cmd, final int startRow, final int perClientRunRows, final int totalRows, final float sampleRate, - boolean flushCommits, boolean writeToWAL, boolean reportLatency, + boolean flushCommits, boolean writeToWAL, boolean reportLatency, int multiGet, HConnection connection, final Status status) throws IOException { status.setStatus("Start " + cmd + " at offset " + startRow + " for " + @@ -1311,7 +1356,7 @@ public class PerformanceEvaluation extends Configured implements Tool { TestOptions options = new TestOptions(startRow, perClientRunRows, totalRows, sampleRate, N, tableName, flushCommits, writeToWAL, - reportLatency, connection); + reportLatency, multiGet, connection); final Test t; try { Constructor constructor = cmd.getDeclaredConstructor( @@ -1345,7 +1390,7 @@ public class PerformanceEvaluation extends Configured implements Tool { admin = new HBaseAdmin(getConf()); checkTable(admin); runOneClient(cmd, 0, this.R, this.R, this.sampleRate, this.flushCommits, - this.writeToWAL, this.reportLatency, this.connection, status); + this.writeToWAL, this.reportLatency, this.multiGet, this.connection, status); } catch (Exception e) { LOG.error("Failed", e); } finally { @@ -1519,6 +1564,12 @@ public class PerformanceEvaluation extends Configured implements Tool { continue; } + final String multiGet = "--multiGet="; + if (cmd.startsWith(multiGet)) { + this.multiGet = Integer.parseInt(cmd.substring(multiGet.length())); + continue; + } + this.connection = HConnectionManager.createConnection(getConf()); Class cmdClass = determineCommandClass(cmd);