From 8ff382aa745ee07f7359c89435f0d24295a5835a Mon Sep 17 00:00:00 2001 From: darionyaphet Date: Tue, 28 Jun 2016 11:23:13 +0800 Subject: [PATCH] HBASE-16104 : Support ColumnFamily in PerformanceEvaluation --- .../hbase/IntegrationTestRegionReplicaPerf.java | 2 +- .../apache/hadoop/hbase/PerformanceEvaluation.java | 90 +++++++++++++++------- .../hbase/mapreduce/TestHFileOutputFormat2.java | 4 +- 3 files changed, 64 insertions(+), 32 deletions(-) diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestRegionReplicaPerf.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestRegionReplicaPerf.java index 8f9b92c..4966910 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestRegionReplicaPerf.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestRegionReplicaPerf.java @@ -236,7 +236,7 @@ public class IntegrationTestRegionReplicaPerf extends IntegrationTestBase { @Override protected Set getColumnFamilies() { - return Sets.newHashSet(Bytes.toString(PerformanceEvaluation.FAMILY_NAME)); + return Sets.newHashSet(PerformanceEvaluation.FAMILY_NAME); } /** Compute the mean of the given {@code stat} from a timing results. */ diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index f673b83..5e4f1d4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -35,6 +35,7 @@ import java.util.Locale; import java.util.Map; import java.util.Queue; import java.util.Random; +import java.util.Set; import java.util.TreeMap; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -129,7 +130,7 @@ public class PerformanceEvaluation extends Configured implements Tool { } public static final String TABLE_NAME = "TestTable"; - public static final byte[] FAMILY_NAME = Bytes.toBytes("info"); + public static final String FAMILY_NAME = "info"; public static final byte [] COLUMN_ZERO = Bytes.toBytes("" + 0); public static final byte [] QUALIFIER_NAME = COLUMN_ZERO; public static final int DEFAULT_VALUE_LENGTH = 1000; @@ -363,6 +364,19 @@ public class PerformanceEvaluation extends Configured implements Tool { } /** + * Check if column family is existed + */ + static boolean checkColumnFamily(Admin admin, TestOptions opts) throws IOException { + HTableDescriptor desc = admin.getTableDescriptor(TableName.valueOf(opts.tableName)); + Set columnFamilies = desc.getFamiliesKeys(); + if (columnFamilies.contains(opts.columnFamily)) { + return true; + } else { + return false; + } + } + + /** * Create an HTableDescriptor from provided TestOptions. */ protected static HTableDescriptor getTableDescriptor(TestOptions opts) { @@ -609,6 +623,7 @@ public class PerformanceEvaluation extends Configured implements Tool { float sampleRate = 1.0f; double traceRate = 0.0; String tableName = TABLE_NAME; + String columnFamily = FAMILY_NAME; boolean flushCommits = true; boolean writeToWAL = true; boolean autoFlush = false; @@ -653,6 +668,7 @@ public class PerformanceEvaluation extends Configured implements Tool { this.sampleRate = that.sampleRate; this.traceRate = that.traceRate; this.tableName = that.tableName; + this.columnFamily = that.columnFamily; this.flushCommits = that.flushCommits; this.writeToWAL = that.writeToWAL; this.autoFlush = that.autoFlush; @@ -785,6 +801,8 @@ public class PerformanceEvaluation extends Configured implements Tool { this.tableName = tableName; } + public void setColumnFamily(String columnFamily) { this.columnFamily = columnFamily; } + public void setFlushCommits(boolean flushCommits) { this.flushCommits = flushCommits; } @@ -889,6 +907,8 @@ public class PerformanceEvaluation extends Configured implements Tool { return tableName; } + public String getColumnFamily() { return columnFamily; } + public boolean isFlushCommits() { return flushCommits; } @@ -1239,9 +1259,9 @@ public class PerformanceEvaluation extends Configured implements Tool { scan.setCaching(opts.caching); FilterList list = new FilterList(); if (opts.addColumns) { - scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); + scan.addColumn(Bytes.toBytes(opts.columnFamily), QUALIFIER_NAME); } else { - scan.addFamily(FAMILY_NAME); + scan.addFamily(Bytes.toBytes(opts.columnFamily)); } if (opts.filterAll) { list.addFilter(new FilterAllFilter()); @@ -1277,9 +1297,9 @@ public class PerformanceEvaluation extends Configured implements Tool { scan.setFilter(new FilterAllFilter()); } if (opts.addColumns) { - scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); + scan.addColumn(Bytes.toBytes(opts.columnFamily), QUALIFIER_NAME); } else { - scan.addFamily(FAMILY_NAME); + scan.addFamily(Bytes.toBytes(opts.columnFamily)); } Result r = null; int count = 0; @@ -1377,9 +1397,9 @@ public class PerformanceEvaluation extends Configured implements Tool { } Get get = new Get(getRandomRow(this.rand, opts.totalRows)); if (opts.addColumns) { - get.addColumn(FAMILY_NAME, QUALIFIER_NAME); + get.addColumn(Bytes.toBytes(opts.columnFamily), QUALIFIER_NAME); } else { - get.addFamily(FAMILY_NAME); + get.addFamily(Bytes.toBytes(opts.columnFamily)); } if (opts.filterAll) { get.setFilter(new FilterAllFilter()); @@ -1433,12 +1453,12 @@ public class PerformanceEvaluation extends Configured implements Tool { Tag t = new ArrayBackedTag((byte) n, tag); tags[n] = t; } - KeyValue kv = new KeyValue(row, FAMILY_NAME, qualifier, HConstants.LATEST_TIMESTAMP, + KeyValue kv = new KeyValue(row, Bytes.toBytes(opts.columnFamily), qualifier, HConstants.LATEST_TIMESTAMP, value, tags); put.add(kv); updateValueSize(kv.getValueLength()); } else { - put.addColumn(FAMILY_NAME, qualifier, value); + put.addColumn(Bytes.toBytes(opts.columnFamily), qualifier, value); updateValueSize(value.length); } } @@ -1469,9 +1489,9 @@ public class PerformanceEvaluation extends Configured implements Tool { Scan scan = new Scan(format(opts.startRow)); scan.setCaching(opts.caching); if (opts.addColumns) { - scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); + scan.addColumn(Bytes.toBytes(opts.columnFamily), QUALIFIER_NAME); } else { - scan.addFamily(FAMILY_NAME); + scan.addFamily(Bytes.toBytes(opts.columnFamily)); } if (opts.filterAll) { scan.setFilter(new FilterAllFilter()); @@ -1521,7 +1541,7 @@ public class PerformanceEvaluation extends Configured implements Tool { @Override void testRow(final int i) throws IOException { Increment increment = new Increment(format(i)); - increment.addColumn(FAMILY_NAME, getQualifier(), 1l); + increment.addColumn(Bytes.toBytes(opts.columnFamily), getQualifier(), 1l); updateValueSize(this.table.increment(increment)); } } @@ -1535,7 +1555,7 @@ public class PerformanceEvaluation extends Configured implements Tool { void testRow(final int i) throws IOException { byte [] bytes = format(i); Append append = new Append(bytes); - append.add(FAMILY_NAME, getQualifier(), bytes); + append.add(Bytes.toBytes(opts.columnFamily), getQualifier(), bytes); updateValueSize(this.table.append(append)); } } @@ -1550,11 +1570,11 @@ public class PerformanceEvaluation extends Configured implements Tool { byte [] bytes = format(i); // Put a known value so when we go to check it, it is there. Put put = new Put(bytes); - put.addColumn(FAMILY_NAME, getQualifier(), bytes); + put.addColumn(Bytes.toBytes(opts.columnFamily), getQualifier(), bytes); this.table.put(put); RowMutations mutations = new RowMutations(bytes); mutations.add(put); - this.table.checkAndMutate(bytes, FAMILY_NAME, getQualifier(), CompareOp.EQUAL, bytes, + this.table.checkAndMutate(bytes, Bytes.toBytes(opts.columnFamily), getQualifier(), CompareOp.EQUAL, bytes, mutations); } } @@ -1569,9 +1589,9 @@ public class PerformanceEvaluation extends Configured implements Tool { byte [] bytes = format(i); // Put a known value so when we go to check it, it is there. Put put = new Put(bytes); - put.addColumn(FAMILY_NAME, getQualifier(), bytes); + put.addColumn(Bytes.toBytes(opts.columnFamily), getQualifier(), bytes); this.table.put(put); - this.table.checkAndPut(bytes, FAMILY_NAME, getQualifier(), CompareOp.EQUAL, bytes, put); + this.table.checkAndPut(bytes, Bytes.toBytes(opts.columnFamily), getQualifier(), CompareOp.EQUAL, bytes, put); } } @@ -1585,11 +1605,11 @@ public class PerformanceEvaluation extends Configured implements Tool { byte [] bytes = format(i); // Put a known value so when we go to check it, it is there. Put put = new Put(bytes); - put.addColumn(FAMILY_NAME, getQualifier(), bytes); + put.addColumn(Bytes.toBytes(opts.columnFamily), getQualifier(), bytes); this.table.put(put); Delete delete = new Delete(put.getRow()); - delete.addColumn(FAMILY_NAME, getQualifier()); - this.table.checkAndDelete(bytes, FAMILY_NAME, getQualifier(), CompareOp.EQUAL, bytes, delete); + delete.addColumn(Bytes.toBytes(opts.columnFamily), getQualifier()); + this.table.checkAndDelete(bytes, Bytes.toBytes(opts.columnFamily), getQualifier(), CompareOp.EQUAL, bytes, delete); } } @@ -1602,7 +1622,7 @@ public class PerformanceEvaluation extends Configured implements Tool { void testRow(final int i) throws IOException { Get get = new Get(format(i)); if (opts.addColumns) { - get.addColumn(FAMILY_NAME, QUALIFIER_NAME); + get.addColumn(Bytes.toBytes(opts.columnFamily), QUALIFIER_NAME); } if (opts.filterAll) { get.setFilter(new FilterAllFilter()); @@ -1630,12 +1650,12 @@ public class PerformanceEvaluation extends Configured implements Tool { Tag t = new ArrayBackedTag((byte) n, tag); tags[n] = t; } - KeyValue kv = new KeyValue(row, FAMILY_NAME, qualifier, HConstants.LATEST_TIMESTAMP, + KeyValue kv = new KeyValue(row, Bytes.toBytes(opts.columnFamily), qualifier, HConstants.LATEST_TIMESTAMP, value, tags); put.add(kv); updateValueSize(kv.getValueLength()); } else { - put.addColumn(FAMILY_NAME, qualifier, value); + put.addColumn(Bytes.toBytes(opts.columnFamily), qualifier, value); updateValueSize(value.length); } } @@ -1669,7 +1689,7 @@ public class PerformanceEvaluation extends Configured implements Tool { protected Scan constructScan(byte[] valuePrefix) throws IOException { FilterList list = new FilterList(); Filter filter = new SingleColumnValueFilter( - FAMILY_NAME, COLUMN_ZERO, CompareFilter.CompareOp.EQUAL, + Bytes.toBytes(opts.columnFamily), COLUMN_ZERO, CompareFilter.CompareOp.EQUAL, new BinaryComparator(valuePrefix) ); list.addFilter(filter); @@ -1679,9 +1699,9 @@ public class PerformanceEvaluation extends Configured implements Tool { Scan scan = new Scan(); scan.setCaching(opts.caching); if (opts.addColumns) { - scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); + scan.addColumn(Bytes.toBytes(opts.columnFamily), QUALIFIER_NAME); } else { - scan.addFamily(FAMILY_NAME); + scan.addFamily(Bytes.toBytes(opts.columnFamily)); } scan.setFilter(list); return scan; @@ -1694,9 +1714,9 @@ public class PerformanceEvaluation extends Configured implements Tool { * @param timeMs Time taken in milliseconds. * @return String value with label, ie '123.76 MB/s' */ - private static String calculateMbps(int rows, long timeMs, final int valueSize, int columns) { + private static String calculateMbps(int rows, String columnFamily, long timeMs, final int valueSize, int columns) { BigDecimal rowSize = BigDecimal.valueOf(ROW_LENGTH + - ((valueSize + FAMILY_NAME.length + COLUMN_ZERO.length) * columns)); + ((valueSize + columnFamily.length() + COLUMN_ZERO.length) * columns)); BigDecimal mbps = BigDecimal.valueOf(rows).multiply(rowSize, CXT) .divide(BigDecimal.valueOf(timeMs), CXT).multiply(MS_PER_SEC, CXT) .divide(BYTES_PER_MB, CXT); @@ -1775,7 +1795,7 @@ public class PerformanceEvaluation extends Configured implements Tool { status.setStatus("Finished " + cmd + " in " + totalElapsedTime + "ms at offset " + opts.startRow + " for " + opts.perClientRunRows + " rows" + - " (" + calculateMbps((int)(opts.perClientRunRows * opts.sampleRate), totalElapsedTime, + " (" + calculateMbps((int)(opts.perClientRunRows * opts.sampleRate), opts.columnFamily, totalElapsedTime, getAverageValueLength(opts), opts.columns) + ")"); return new RunResult(totalElapsedTime, t.getLatencyHistogram()); @@ -1796,6 +1816,11 @@ public class PerformanceEvaluation extends Configured implements Tool { connection = ConnectionFactory.createConnection(getConf()); admin = connection.getAdmin(); checkTable(admin, opts); + if (!checkColumnFamily(admin, opts)) { + HColumnDescriptor descriptor = new HColumnDescriptor(opts.columnFamily); + LOG.info("Create Column Family : " + opts.columnFamily); + admin.addColumnFamily(TableName.valueOf(opts.tableName), descriptor); + } } finally { if (admin != null) admin.close(); if (connection != null) connection.close(); @@ -1850,6 +1875,7 @@ public class PerformanceEvaluation extends Configured implements Tool { System.err.println(); System.err.println("Table Creation / Write Tests:"); System.err.println(" table Alternate table name. Default: 'TestTable'"); + System.err.println(" columnFamily Alternate table name. Default: 'info'"); System.err.println(" rows Rows each client runs. Default: " + DEFAULT_OPTS.getPerClientRunRows()); System.err.println(" size Total size in GiB. Mutually exclusive with --rows. " + @@ -1956,6 +1982,12 @@ public class PerformanceEvaluation extends Configured implements Tool { continue; } + final String columnFamily = "--columnFamily="; + if (cmd.startsWith(columnFamily)) { + opts.columnFamily = cmd.substring(columnFamily.length()); + continue; + } + final String startRow = "--startRow="; if (cmd.startsWith(startRow)) { opts.startRow = Integer.parseInt(cmd.substring(startRow.length())); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java index 486c961..1edadc5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java @@ -118,8 +118,8 @@ public class TestHFileOutputFormat2 { private final static int ROWSPERSPLIT = 1024; private static final byte[][] FAMILIES - = { Bytes.add(PerformanceEvaluation.FAMILY_NAME, Bytes.toBytes("-A")) - , Bytes.add(PerformanceEvaluation.FAMILY_NAME, Bytes.toBytes("-B"))}; + = { Bytes.add(Bytes.toBytes(PerformanceEvaluation.FAMILY_NAME), Bytes.toBytes("-A")) + , Bytes.add(Bytes.toBytes(PerformanceEvaluation.FAMILY_NAME), Bytes.toBytes("-B"))}; private static final TableName TABLE_NAME = TableName.valueOf("TestTable"); -- 2.9.0