Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestJoinedScanners.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestJoinedScanners.java (revision 1466089) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestJoinedScanners.java (working copy) @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -40,12 +41,12 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.CompareFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Test; import org.junit.experimental.categories.Category; - /** * Test performance improvement of joined scanners optimization: * https://issues.apache.org/jira/browse/HBASE-5416 @@ -63,6 +64,10 @@ private static final byte[] flag_yes = Bytes.toBytes("Y"); private static final byte[] flag_no = Bytes.toBytes("N"); + private static DataBlockEncoding blockEncoding = DataBlockEncoding.FAST_DIFF; + private static int selectionRatio = 30; + private static int valueWidth = 128 * 1024; + @Test public void testJoinedScanners() throws Exception { String dataNodeHosts[] = new String[] { "host1", "host2", "host3" }; @@ -80,28 +85,33 @@ cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts); byte [][] families = {cf_essential, cf_joined}; - HTable ht = htu.createTable( - Bytes.toBytes(this.getClass().getSimpleName()), families); + byte[] tableName = Bytes.toBytes(this.getClass().getSimpleName()); + HTableDescriptor desc = new HTableDescriptor(tableName); + for(byte[] family : families) { + HColumnDescriptor hcd = new HColumnDescriptor(family); + hcd.setDataBlockEncoding(blockEncoding); + desc.addFamily(hcd); + } + htu.getHBaseAdmin().createTable(desc); + HTable ht = new HTable(htu.getConfiguration(), tableName); long rows_to_insert = 10000; int insert_batch = 20; - int flag_percent = 1; - int large_bytes = 128 * 1024; long time = System.nanoTime(); + Random rand = new Random(time); LOG.info("Make " + Long.toString(rows_to_insert) + " rows, total size = " - + Float.toString(rows_to_insert * large_bytes / 1024 / 1024) + " MB"); + + Float.toString(rows_to_insert * valueWidth / 1024 / 1024) + " MB"); - byte [] val_large = new byte[large_bytes]; + byte [] val_large = new byte[valueWidth]; List puts = new ArrayList(); for (long i = 0; i < rows_to_insert; i++) { Put put = new Put(Bytes.toBytes(Long.toString (i))); - if (i % 100 <= flag_percent) { + if (rand.nextInt(100) <= selectionRatio) { put.add(cf_essential, col_name, flag_yes); - } - else { + } else { put.add(cf_essential, col_name, flag_no); } put.add(cf_joined, col_name, val_large); @@ -163,7 +173,9 @@ throws IOException { HTableDescriptor htd = new HTableDescriptor(tableName); for(byte [] family : families) { - htd.addFamily(new HColumnDescriptor(family)); + HColumnDescriptor hcd = new HColumnDescriptor(family); + hcd.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF); + htd.addFamily(hcd); } HRegionInfo info = new HRegionInfo(htd.getName(), startKey, stopKey, false); Path path = new Path(DIR + callingMethod); @@ -175,4 +187,45 @@ } return HRegion.createHRegion(info, path, conf, htd); } + + protected static void printUsage(final String message) { + if (message != null && message.length() > 0) { + System.err.println(message); + } + System.err.println("Usage: java TestJoinedScanners \\"); + System.err.println(" [--blockEncoding=TYPE] [--selectionRatio=ratio] [--valueWidth="); + System.err.println(); + System.err.println("Options:"); + System.err.println(" blockEncoding Data block encoding; Default: FAST_DIFF"); + System.err.println(" selectionRatio Ratio of selected rows using essential column family"); + System.err.println(" valueWidth Width of value for non-essential column family"); + } + + /** + * Command line interface: + * @param args + * @throws IOException if there is a bug while reading from disk + */ + public static void main(final String[] args) throws IOException { + if (args.length < 1) { + printUsage(null); + System.exit(-1); + } + for (int i = 0; i < args.length; i++) { + String cmd = args[i]; + final String blkEncoding = "--blockEncoding="; + if (cmd.startsWith(blkEncoding)) { + blockEncoding = DataBlockEncoding.valueOf(cmd.substring(blkEncoding.length())); + continue; + } + final String ratio = "--selectionRatio="; + if (cmd.startsWith(ratio)) { + selectionRatio = Integer.parseInt(cmd.substring(ratio.length())); + } + final String width = "--valueWidth="; + if (cmd.startsWith(width)) { + valueWidth = Integer.parseInt(cmd.substring(width.length())); + } + } + } } \ No newline at end of file