Index: src/test/java/org/apache/hadoop/hbase/HLogPerformanceEvaluation.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/HLogPerformanceEvaluation.java (revision 0) +++ src/test/java/org/apache/hadoop/hbase/HLogPerformanceEvaluation.java (revision 0) @@ -0,0 +1,247 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.util.Map; +import java.util.List; +import java.util.Random; +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.classification.InterfaceAudience; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; + +/** + *

+ * This class runs performance benchmarks for {@link HLog}. + *

+ */ +@InterfaceAudience.Private +public final class HLogPerformanceEvaluation extends Configured implements Tool { + static final Log LOG = LogFactory.getLog(HLogPerformanceEvaluation.class.getName()); + + private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + static final String TABLE_NAME = "HLogPerformanceEvaluation"; + static final String QUALIFIER_PREFIX = "q"; + static final String FAMILY_PREFIX = "cf"; + + private HTableDescriptor htd = null; + private HRegionInfo regionInfo = null; + private HRegion region = null; + + private Path regionRootDir = TEST_UTIL.getDataTestDir("HLogPerformanceEvaluation"); + private long numIterations = 10000; + private int numThreads = 1; + private boolean noSync = false; + private int numQualifiers = 1; + private int numFamilies = 1; + private int valueSize = 512; + private int keySize = 16; + + private FileSystem fs = null; + + /** + * Perform HLog.append() of Put object, for the number of iterations requested. + * Keys and Vaues are generated randomly, the number of column familes, + * qualifiers and key/value size is tunable by the user. + */ + class HLogPutBenchmark implements Runnable { + public void run() { + byte[] key = new byte[keySize]; + byte[] value = new byte[valueSize]; + Random rand = new Random(Thread.currentThread().getId()); + HLog hlog = region.getLog(); + + try { + long startTime = System.currentTimeMillis(); + for (int i = 0; i < numIterations; ++i) { + Put put = setupPut(rand, key, value); + + long now = System.currentTimeMillis(); + WALEdit walEdit = new WALEdit(); + addFamilyMapToWALEdit(put.getFamilyMap(), walEdit); + + if (noSync) { + hlog.appendNoSync(regionInfo, htd.getName(), walEdit, + HConstants.DEFAULT_CLUSTER_ID, now, htd); + } else { + hlog.append(regionInfo, htd.getName(), walEdit, now, htd); + } + } + long totalTime = (System.currentTimeMillis() - startTime); + logBenchmarkResult(getClass().getSimpleName() + " Thread", numIterations, totalTime); + } catch (Exception e) { + LOG.error(getClass().getSimpleName() + " Thread failed", e); + } + } + } + + @Override + public int run(String[] args) throws Exception { + // Process command line args + for (int i = 0; i < args.length; i++) { + String cmd = args[i]; + + try { + if (cmd.equals("-threads")) { + numThreads = Integer.parseInt(args[++i]); + } else if (cmd.equals("-iterations")) { + numIterations = Long.parseLong(args[++i]); + } else if (cmd.equals("-path")) { + regionRootDir = new Path(args[++i]); + } else if (cmd.equals("-families")) { + numFamilies = Integer.parseInt(args[++i]); + } else if (cmd.equals("-qualifiers")) { + numQualifiers = Integer.parseInt(args[++i]); + } else if (cmd.equals("-keySize")) { + keySize = Integer.parseInt(args[++i]); + } else if (cmd.equals("-valueSize")) { + valueSize = Integer.parseInt(args[++i]); + } else if (cmd.equals("-nosync")) { + noSync = true; + } else { + printUsageAndExit(); + } + } catch (Exception e) { + printUsageAndExit(); + } + } + + // Run HLog Performance Evaluation + fs = FileSystem.get(getConf()); + try { + regionRootDir = regionRootDir.makeQualified(this.fs); + cleanRegionRootDir(); + try { + openRegion(); + + long putTime = runBenchmark(new HLogPutBenchmark()); + logBenchmarkResult("HLogPutBenchmark", numIterations * numThreads, putTime); + } finally { + closeRegion(); + } + } finally { + fs.close(); + } + + return(0); + } + + private static void logBenchmarkResult(String testName, long numTests, long totalTime) { + float tsec = totalTime / 1000.0f; + LOG.info(String.format("%s took %.3fsec %.3fops/sec", testName, tsec, numTests / tsec)); + } + + private void printUsageAndExit() { + System.err.printf("Usage: bin/hbase %s [opts]\n", getClass().getName()); + System.err.println(" where [opts] are:"); + System.err.println(" -help Show this help and exit."); + System.err.println(" -threads Number of threads writing on the WAL."); + System.err.println(" -iterations Number of iterations per thread."); + System.err.println(" -path Path where region's root directory is created."); + System.err.println(" -families Number of column families to write."); + System.err.println(" -qualifiers Number of qualifiers to write."); + System.err.println(" -keySize Row key size in byte."); + System.err.println(" -valueSize Row/Col value size in byte."); + System.err.println(" -nosync Append without syncing"); + System.exit(1); + } + + private void openRegion() throws IOException { + cleanRegionRootDir(); + + // Initialize Table Descriptor + htd = new HTableDescriptor(TABLE_NAME); + for (int i = 0; i < numFamilies; ++i) { + HColumnDescriptor colDef = new HColumnDescriptor(FAMILY_PREFIX + i); + htd.addFamily(colDef); + } + + // Initialize HRegion + regionInfo = new HRegionInfo(htd.getName()); + region = HRegion.createHRegion(regionInfo, regionRootDir, getConf(), htd); + } + + private void closeRegion() throws IOException { + if (region != null) { + region.close(); + region = null; + + cleanRegionRootDir(); + } + } + + private void cleanRegionRootDir() throws IOException { + if (fs.exists(regionRootDir)) { + fs.delete(regionRootDir, true); + } + } + + private Put setupPut(Random rand, byte[] key, byte[] value) { + rand.nextBytes(key); + Put put = new Put(key); + for (int cf = 0; cf < numFamilies; ++cf) { + for (int q = 0; q < numQualifiers; ++q) { + rand.nextBytes(value); + put.add(Bytes.toBytes(FAMILY_PREFIX + cf), Bytes.toBytes(QUALIFIER_PREFIX + q), value); + } + } + return put; + } + + private void addFamilyMapToWALEdit(Map> familyMap, WALEdit walEdit) { + for (List edits : familyMap.values()) { + for (KeyValue kv : edits) { + walEdit.add(kv); + } + } + } + + private long runBenchmark(Runnable runnable) throws InterruptedException { + Thread[] threads = new Thread[numThreads]; + + long startTime = System.currentTimeMillis(); + for (int i = 0; i < numThreads; ++i) { + threads[i] = new Thread(runnable); + threads[i].start(); + } + for (Thread t : threads) t.join(); + long endTime = System.currentTimeMillis(); + return(endTime - startTime); + } + + public static void main(String[] args) throws Exception { + int exitCode = ToolRunner.run(new HLogPerformanceEvaluation(), args); + System.exit(exitCode); + } +}