diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java index 1370415..9641f72 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java @@ -64,6 +64,7 @@ import org.apache.hadoop.hbase.mapreduce.TableMapper; import org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl; import org.apache.hadoop.hbase.util.AbstractHBaseTool; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.LoadTestTool; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; @@ -87,6 +88,7 @@ import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -163,7 +165,7 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { protected static String DEFAULT_TABLE_NAME = "IntegrationTestBigLinkedList"; - private static byte[] FAMILY_NAME = Bytes.toBytes("meta"); + private static byte[] FAMILY_NAME = Bytes.toBytes("test_cf"); //link to the id of the prev node in the linked list private static final byte[] COLUMN_PREV = Bytes.toBytes("prev"); @@ -234,6 +236,7 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { private long count; private long numNodes; private Random rand; + byte[][] startKeys; @Override public void close() throws IOException { @@ -241,9 +244,17 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { @Override public BytesWritable getCurrentKey() throws IOException, InterruptedException { - byte[] bytes = new byte[ROWKEY_LENGTH]; - rand.nextBytes(bytes); - return new BytesWritable(bytes); + byte[] suffix = new byte[ROWKEY_LENGTH/2]; + rand.nextBytes(suffix); + int targetRegion = rand.nextInt(startKeys.length); + byte[] key = new byte[ROWKEY_LENGTH]; + if (startKeys[targetRegion].length >= ROWKEY_LENGTH / 2) { + System.arraycopy(startKeys[targetRegion], 0, key, 0, ROWKEY_LENGTH / 2); + System.arraycopy(suffix, 0, key, ROWKEY_LENGTH / 2, ROWKEY_LENGTH / 2); + } else { + rand.nextBytes(key); + } + return new BytesWritable(key); } @Override @@ -261,6 +272,10 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { throws IOException, InterruptedException { numNodes = context.getConfiguration().getLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, 25000000); rand = new Random(); + Configuration conf = context.getConfiguration(); + HTable table = new HTable(conf, getTableName(conf)); + startKeys = table.getStartKeys(); + table.close(); } @Override @@ -440,14 +455,11 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { } protected void createSchema() throws IOException { - HBaseAdmin admin = new HBaseAdmin(getConf()); TableName tableName = getTableName(getConf()); - if (!admin.tableExists(tableName)) { - HTableDescriptor htd = new HTableDescriptor(getTableName(getConf())); - htd.addFamily(new HColumnDescriptor(FAMILY_NAME)); - admin.createTable(htd); - } - admin.close(); + LoadTestTool loadTool = new LoadTestTool(); + loadTool.setConf(getConf()); + int ret = loadTool.run(new String[] { "-tn", tableName.getNameAsString(), "-init_only" }); + Assert.assertEquals("Failed to initialize LoadTestTool", 0, ret); } public int runRandomInputGenerator(int numMappers, long numNodes, Path tmpOutput, @@ -463,6 +475,9 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { job.setInputFormatClass(GeneratorInputFormat.class); job.setOutputKeyClass(BytesWritable.class); job.setOutputValueClass(NullWritable.class); + TableMapReduceUtil.addDependencyJars(job); + TableMapReduceUtil.addDependencyJars(job.getConfiguration(), HTable.class); + TableMapReduceUtil.initCredentials(job); setJobConf(job, numMappers, numNodes, width, wrapMuplitplier); @@ -479,7 +494,6 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { public int runGenerator(int numMappers, long numNodes, Path tmpOutput, Integer width, Integer wrapMuplitplier) throws Exception { LOG.info("Running Generator with numMappers=" + numMappers +", numNodes=" + numNodes); - createSchema(); Job job = new Job(getConf()); @@ -510,6 +524,7 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { public int run(int numMappers, long numNodes, Path tmpOutput, Integer width, Integer wrapMuplitplier) throws Exception { + createSchema(); int ret = runRandomInputGenerator(numMappers, numNodes, tmpOutput, width, wrapMuplitplier); if (ret > 0) { return ret;