Index: src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java =================================================================== --- src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java (revision a47d0fc5c598d412d1a9808da34dc33a213f6fc8) +++ src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java Tue Mar 09 12:35:56 EST 2010 @@ -24,7 +24,6 @@ import java.io.IOException; import java.io.PrintStream; import java.io.File; -import java.lang.reflect.InvocationTargetException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; @@ -119,7 +118,7 @@ private boolean nomapred = false; private int N = 1; private int R = ROWS_PER_GB; - private boolean flushCommits = false; + private boolean flushTable = false; private static final Path PERF_EVAL_DIR = new Path("performance_evaluation"); /** @@ -127,7 +126,7 @@ */ public static final Pattern LINE_PATTERN = Pattern.compile("startRow=(\\d+),\\s+" + - "perClientRunRows=(\\d+),\\s+totalRows=(\\d+),\\s+clients=(\\d+),\\s+flushCommits=(\\w+)"); + "perClientRunRows=(\\d+),\\s+totalRows=(\\d+),\\s+clients=(\\d+),\\s+flushTable=(\\w+)"); /** * Enum for map metrics. Keep it out here rather than inside in the Map @@ -199,22 +198,22 @@ private int rows = 0; private int totalRows = 0; private int clients = 0; - private boolean flushCommits = false; + private boolean flushTable = false; - + public PeInputSplit() { this.startRow = 0; this.rows = 0; this.totalRows = 0; this.clients = 0; - this.flushCommits = false; + this.flushTable = false; } - public PeInputSplit(int startRow, int rows, int totalRows, int clients, boolean flushCommits) { + public PeInputSplit(int startRow, int rows, int totalRows, int clients, boolean flushTable) { this.startRow = startRow; this.rows = rows; this.totalRows = totalRows; this.clients = clients; - this.flushCommits = flushCommits; + this.flushTable = flushTable; } @Override @@ -223,7 +222,7 @@ this.rows = in.readInt(); this.totalRows = in.readInt(); this.clients = in.readInt(); - this.flushCommits = in.readBoolean(); + this.flushTable = in.readBoolean(); } @Override @@ -232,7 +231,7 @@ out.writeInt(rows); out.writeInt(totalRows); out.writeInt(clients); - out.writeBoolean(flushCommits); + out.writeBoolean(flushTable); } @Override @@ -261,8 +260,8 @@ return clients; } - public boolean isFlushCommits() { - return flushCommits; + public boolean isFlushTable() { + return flushTable; } } @@ -295,16 +294,16 @@ int rows = Integer.parseInt(m.group(2)); int totalRows = Integer.parseInt(m.group(3)); int clients = Integer.parseInt(m.group(4)); - boolean flushCommits = Boolean.parseBoolean(m.group(5)); + boolean flushTable = Boolean.parseBoolean(m.group(5)); LOG.debug("split["+ splitList.size() + "] " + " startRow=" + startRow + " rows=" + rows + " totalRows=" + totalRows + " clients=" + clients + - " flushCommits=" + flushCommits); + " flushTable=" + flushTable); - PeInputSplit newSplit = new PeInputSplit(startRow, rows, totalRows, clients, flushCommits); + PeInputSplit newSplit = new PeInputSplit(startRow, rows, totalRows, clients, flushTable); splitList.add(newSplit); } } @@ -424,7 +423,7 @@ // Evaluation task long elapsedTime = this.pe.runOneClient(this.cmd, value.getStartRow(), - value.getRows(), value.getTotalRows(), value.isFlushCommits(), status); + value.getRows(), value.getTotalRows(), value.isFlushTable(), status); // Collect how much time the thing took. Report as map output and // to the ELAPSED_TIME counter. context.getCounter(Counter.ELAPSED_TIME).increment(elapsedTime); @@ -489,7 +488,7 @@ try { long elapsedTime = pe.runOneClient(cmd, index * perClientRows, perClientRows, R, - flushCommits, new Status() { + flushTable, new Status() { public void setStatus(final String msg) throws IOException { LOG.info("client-" + getName() + " " + msg); } @@ -577,7 +576,7 @@ ", perClientRunRows=" + (perClientRows / 10) + ", totalRows=" + this.R + ", clients=" + this.N + - ", flushCommits=" + this.flushCommits; + ", flushTable=" + this.flushTable; int hash = h.hash(Bytes.toBytes(s)); m.put(hash, s); } @@ -627,17 +626,17 @@ private int perClientRunRows; private int totalRows; private byte[] tableName; - private boolean flushCommits; + private boolean flushTable; TestOptions() { } - TestOptions(int startRow, int perClientRunRows, int totalRows, byte[] tableName, boolean flushCommits) { + TestOptions(int startRow, int perClientRunRows, int totalRows, byte[] tableName, boolean flushTable) { this.startRow = startRow; this.perClientRunRows = perClientRunRows; this.totalRows = totalRows; this.tableName = tableName; - this.flushCommits = flushCommits; + this.flushTable = flushTable; } public int getStartRow() { @@ -656,8 +655,8 @@ return tableName; } - public boolean isFlushCommits() { - return flushCommits; + public boolean isFlushTable() { + return flushTable; } } @@ -675,7 +674,7 @@ protected HBaseAdmin admin; protected HTable table; protected volatile HBaseConfiguration conf; - protected boolean flushCommits; + protected boolean flushTable; /** * Note that all subclasses of this class must provide a public contructor @@ -690,7 +689,7 @@ this.tableName = options.getTableName(); this.table = null; this.conf = conf; - this.flushCommits = options.isFlushCommits(); + this.flushTable = options.isFlushTable(); } private String generateStatus(final int sr, final int i, final int lr) { @@ -711,8 +710,8 @@ } void testTakedown() throws IOException { - if (flushCommits) { - this.table.flushCommits(); + if (flushTable) { + new HBaseAdmin(conf).flush(this.tableName); } } @@ -746,6 +745,9 @@ status.setStatus(generateStatus(this.startRow, i, lastRow)); } } + + // flush any pending puts before calculating the elapsed time + this.table.flushCommits(); } /* @@ -1029,7 +1031,7 @@ } long runOneClient(final Class cmd, final int startRow, - final int perClientRunRows, final int totalRows, boolean flushCommits, final Status status) + final int perClientRunRows, final int totalRows, boolean flushTable, final Status status) throws IOException { status.setStatus("Start " + cmd + " at offset " + startRow + " for " + perClientRunRows + " rows"); @@ -1037,7 +1039,7 @@ Test t = null; TestOptions options = new TestOptions(startRow, perClientRunRows, - totalRows, getTableDescriptor().getName(), flushCommits); + totalRows, getTableDescriptor().getName(), flushTable); try { Constructor constructor = cmd.getDeclaredConstructor( HBaseConfiguration.class, @@ -1071,7 +1073,7 @@ try { admin = new HBaseAdmin(this.conf); checkTable(admin); - runOneClient(cmd, 0, this.R, this.R, this.flushCommits, status); + runOneClient(cmd, 0, this.R, this.R, this.flushTable, status); } catch (Exception e) { LOG.error("Failed", e); } @@ -1126,18 +1128,21 @@ System.err.println(message); } System.err.println("Usage: java " + this.getClass().getName() + " \\"); - System.err.println(" [--miniCluster] [--nomapred] [--rows=ROWS] "); + System.err.println(" [--miniCluster] [--nomapred] [--rows=ROWS] " + + "[--flushTable=true|false] "); System.err.println(); System.err.println("Options:"); System.err.println(" miniCluster Run the test on an HBaseMiniCluster"); System.err.println(" nomapred Run multiple clients using threads " + "(rather than use mapreduce)"); System.err.println(" rows Rows each client runs. Default: One million"); - System.err.println(" flushCommits Used to determine if the test should flush the table. Default: false"); + System.err.println(" flushTable Used to determine if the test should " + + "flush the table. Default: false"); System.err.println(); System.err.println("Command:"); for (CmdDescriptor command : commands.values()) { - System.err.println(String.format(" %-15s %s", command.getName(), command.getDescription())); + System.err.println(String.format(" %-15s %s", command.getName(), + command.getDescription())); } System.err.println(); System.err.println("Args:"); @@ -1198,9 +1203,9 @@ continue; } - final String flushCommits = "--flushCommits="; - if (cmd.startsWith(rows)) { - this.flushCommits = Boolean.parseBoolean(cmd.substring(flushCommits.length())); + final String flushTable = "--flushTable="; + if (cmd.startsWith(flushTable)) { + this.flushTable = Boolean.parseBoolean(cmd.substring(flushTable.length())); continue; }