diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java index b696f41..d89bd05 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java @@ -684,4 +684,4 @@ public final class CellUtil { } return commonPrefix; } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index b367a0d..f963155 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -202,29 +202,6 @@ public class HFile { void append(Cell cell) throws IOException; - /** - * @param key - * @param value - * @throws IOException - * @Deprecated Use {@link #append(Cell)} - */ - // This method makes no sense anymore. What is 'key'? Row? How can you have a 'row' w/o - // a family, etc. - @Deprecated - void append(byte[] key, byte[] value) throws IOException; - - // This method makes no sense anymore. What is 'key'? Row? How can you have a 'row' w/o - // a family, etc. - /** - * @param key - * @param value - * @param tag - * @throws IOException - * @Deprecated Use {@link #append(Cell)} - */ - @Deprecated - void append (byte[] key, byte[] value, byte[] tag) throws IOException; - /** @return the path to this {@link HFile} */ Path getPath(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java index b951fab..4be14b5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java @@ -60,7 +60,7 @@ public interface HFileScanner { @Deprecated int seekTo(byte[] key, int offset, int length) throws IOException; - int seekTo(Cell kv) throws IOException; + int seekTo(Cell c) throws IOException; /** * Reseek to or just before the passed key. Similar to seekTo * except that this can be called even if the scanner is not at the beginning @@ -86,7 +86,7 @@ public interface HFileScanner { @Deprecated int reseekTo(byte[] key, int offset, int length) throws IOException; - int reseekTo(Cell kv) throws IOException; + int reseekTo(Cell c) throws IOException; /** * Consider the key stream of all the keys in the file, * k[0] .. k[n], where there are n keys in the file. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java index ee0e303..49211c8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.io.hfile.HFile.Writer; @@ -244,16 +243,22 @@ public class HFileWriterV2 extends AbstractHFileWriter { * Add key/value to file. Keys must be added in an order that agrees with the * Comparator passed on construction. * - * @param cell - * Cell to add. Cannot be empty nor null. + * @param cell Cell to add. Cannot be empty nor null. * @throws IOException */ @Override public void append(final Cell cell) throws IOException { + // TODO: + // + Need to get a key from Cell to add to the index, to use as first key in block. + // It must work with passed in KVComparator. + // + For index, always needs to be in order? Always needs to be in order. + // Q: Can DBE destroy order? + // Should DBE be making the index key? It won't work with KVComparator if it does. int klength = KeyValueUtil.keyLength(cell); byte[] value = cell.getValueArray(); int voffset = cell.getValueOffset(); int vlength = cell.getValueLength(); + // checkKey uses comparator to check we are writing in order. boolean dupKey = checkKey(cell); checkValue(value, voffset, vlength); if (!dupKey) { @@ -270,7 +275,7 @@ public class HFileWriterV2 extends AbstractHFileWriter { // Are we the first key in this block? if (firstKeyInBlock == null) { - // Copy the key. + // Copy the key for use as first key in block. It is put into file index. firstKeyInBlock = new byte[klength]; KeyValueUtil.appendKeyTo(cell, firstKeyInBlock, 0); } @@ -281,28 +286,6 @@ public class HFileWriterV2 extends AbstractHFileWriter { this.maxMemstoreTS = Math.max(this.maxMemstoreTS, cell.getSequenceId()); } - /** - * Add key/value to file. Keys must be added in an order that agrees with the - * Comparator passed on construction. - * - * @param key - * Key to add. Cannot be empty nor null. - * @param value - * Value to add. Cannot be empty nor null. - * @throws IOException - */ - @Override - public void append(final byte[] key, final byte[] value) throws IOException { - int kvlen = (int) KeyValue.getKeyValueDataStructureSize(key.length, value.length, 0); - byte[] b = new byte[kvlen]; - int pos = 0; - pos = Bytes.putInt(b, pos, key.length); - pos = Bytes.putInt(b, pos, value.length); - pos = Bytes.putBytes(b, pos, key, 0, key.length); - Bytes.putBytes(b, pos, value, 0, value.length); - append(new KeyValue(b, 0, kvlen)); - } - @Override public void close() throws IOException { if (outputStream == null) { @@ -426,11 +409,6 @@ public class HFileWriterV2 extends AbstractHFileWriter { }); } - @Override - public void append(byte[] key, byte[] value, byte[] tag) throws IOException { - throw new UnsupportedOperationException("KV tags are supported only from HFile V3"); - } - protected int getMajorVersion() { return 2; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV3.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV3.java index a0a2372..9afe372 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV3.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV3.java @@ -28,7 +28,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; @@ -91,47 +90,6 @@ public class HFileWriterV3 extends HFileWriterV2 { this.maxTagsLength = tagsLength; } } - - /** - * Add key/value to file. Keys must be added in an order that agrees with the - * Comparator passed on construction. - * @param key - * Key to add. Cannot be empty nor null. - * @param value - * Value to add. Cannot be empty nor null. - * @throws IOException - */ - @Override - public void append(final byte[] key, final byte[] value) throws IOException { - append(key, value, HConstants.EMPTY_BYTE_ARRAY); - } - - /** - * Add key/value to file. Keys must be added in an order that agrees with the - * Comparator passed on construction. - * @param key - * Key to add. Cannot be empty nor null. - * @param value - * Value to add. Cannot be empty nor null. - * @param tag - * Tag t add. Cannot be empty or null. - * @throws IOException - */ - @Override - public void append(final byte[] key, final byte[] value, byte[] tag) throws IOException { - int kvlen = (int) KeyValue.getKeyValueDataStructureSize(key.length, value.length, tag.length); - byte[] b = new byte[kvlen]; - int pos = 0; - pos = Bytes.putInt(b, pos, key.length); - pos = Bytes.putInt(b, pos, value.length); - pos = Bytes.putBytes(b, pos, key, 0, key.length); - pos = Bytes.putBytes(b, pos, value, 0, value.length); - if (tag.length > 0) { - pos = Bytes.putAsShort(b, pos, tag.length); - Bytes.putBytes(b, pos, tag, 0, tag.length); - } - append(new KeyValue(b, 0, kvlen)); - } protected void finishFileInfo() throws IOException { super.finishFileInfo(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java index a1dca6f..7f2ffee 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java @@ -38,12 +38,9 @@ import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.util.Bytes; /** - *

* This class runs performance benchmarks for {@link HFile}. - *

*/ public class HFilePerformanceEvaluation { - private static final int ROW_LENGTH = 10; private static final int ROW_COUNT = 1000000; private static final int RFILE_BLOCKSIZE = 8 * 1024; @@ -61,6 +58,30 @@ public class HFilePerformanceEvaluation { return w; } + static Cell createCell(final int i) { + return createCell(i, HConstants.EMPTY_BYTE_ARRAY); + } + + /** + * HFile is Cell-based. It used to be byte arrays. Doing this test, pass Cells. All Cells + * intentionally have same coordinates in all fields but row. + * @param i Integer to format as a row Key. + * @param value Value to use + * @return Created Cell. + */ + static Cell createCell(final int i, final byte [] value) { + return createCell(format(i), value); + } + + static Cell createCell(final byte [] keyRow) { + return createCell(keyRow, HConstants.EMPTY_BYTE_ARRAY); + } + + static Cell createCell(final byte [] keyRow, final byte [] value) { + return CellUtil.createCell(keyRow, HConstants.CATALOG_FAMILY, HConstants.EMPTY_BYTE_ARRAY, + HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum.getCode(), value); + } + private void runBenchmarks() throws Exception { final Configuration conf = new Configuration(); final FileSystem fs = FileSystem.get(conf); @@ -200,10 +221,7 @@ public class HFilePerformanceEvaluation { @Override void doRow(int i) throws Exception { - Cell c = CellUtil.createCell(format(i), HConstants.CATALOG_FAMILY, - HConstants.CATALOG_FAMILY, HConstants.LATEST_TIMESTAMP, KeyValue.Type.Minimum.getCode(), - generateValue()); - writer.append(c); + writer.append(createCell(i, generateValue())); } private byte[] generateValue() { @@ -263,10 +281,10 @@ public class HFilePerformanceEvaluation { @Override void doRow(int i) throws Exception { if (this.scanner.next()) { - ByteBuffer k = this.scanner.getKey(); - PerformanceEvaluationCommons.assertKey(format(i + 1), k); - ByteBuffer v = scanner.getValue(); - PerformanceEvaluationCommons.assertValueSize(v.limit(), ROW_LENGTH); + // TODO: Fix. Make Scanner do Cells. + Cell c = this.scanner.getKeyValue(); + PerformanceEvaluationCommons.assertKey(format(i + 1), c); + PerformanceEvaluationCommons.assertValueSize(c.getValueLength(), ROW_LENGTH); } } @@ -290,14 +308,14 @@ public class HFilePerformanceEvaluation { void doRow(int i) throws Exception { HFileScanner scanner = this.reader.getScanner(false, true); byte [] b = getRandomRow(); - if (scanner.seekTo(b) < 0) { + if (scanner.seekTo(createCell(b)) < 0) { LOG.info("Not able to seekTo " + new String(b)); return; } - ByteBuffer k = scanner.getKey(); - PerformanceEvaluationCommons.assertKey(b, k); - ByteBuffer v = scanner.getValue(); - PerformanceEvaluationCommons.assertValueSize(v.limit(), ROW_LENGTH); + // TODO: Fix scanner so it does Cells + Cell c = scanner.getKeyValue(); + PerformanceEvaluationCommons.assertKey(b, c); + PerformanceEvaluationCommons.assertValueSize(c.getValueLength(), ROW_LENGTH); } private byte [] getRandomRow() { @@ -317,20 +335,24 @@ public class HFilePerformanceEvaluation { void doRow(int i) throws Exception { HFileScanner scanner = this.reader.getScanner(false, false); byte [] b = getRandomRow(); - if (scanner.seekTo(b) != 0) { + // System.out.println("Random row: " + new String(b)); + Cell c = createCell(b); + if (scanner.seekTo(c) != 0) { LOG.info("Nonexistent row: " + new String(b)); return; } - ByteBuffer k = scanner.getKey(); - PerformanceEvaluationCommons.assertKey(b, k); - // System.out.println("Found row: " + new String(b)); + // TODO: HFileScanner doesn't do Cells yet. Temporary fix. + c = scanner.getKeyValue(); + // System.out.println("Found row: " + + // new String(c.getRowArray(), c.getRowOffset(), c.getRowLength())); + PerformanceEvaluationCommons.assertKey(b, c); for (int ii = 0; ii < 30; ii++) { if (!scanner.next()) { LOG.info("NOTHING FOLLOWS"); return; } - ByteBuffer v = scanner.getValue(); - PerformanceEvaluationCommons.assertValueSize(v.limit(), ROW_LENGTH); + c = scanner.getKeyValue(); + PerformanceEvaluationCommons.assertValueSize(c.getValueLength(), ROW_LENGTH); } } @@ -352,14 +374,14 @@ public class HFilePerformanceEvaluation { void doRow(int i) throws Exception { HFileScanner scanner = this.reader.getScanner(false, true); byte[] gaussianRandomRowBytes = getGaussianRandomRowBytes(); - scanner.seekTo(gaussianRandomRowBytes); + scanner.seekTo(createCell(gaussianRandomRowBytes)); for (int ii = 0; ii < 30; ii++) { if (!scanner.next()) { LOG.info("NOTHING FOLLOWS"); return; } - scanner.getKey(); - scanner.getValue(); + // TODO: Fix. Make scanner do Cells. + scanner.getKeyValue(); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java index 6ac9865..4b0aa58 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java @@ -45,11 +45,22 @@ public class PerformanceEvaluationCommons { assertKey(expected, b); } + public static void assertKey(final byte [] expected, final Cell c) { + assertKey(expected, c.getRowArray(), c.getRowOffset(), c.getRowLength()); + } + public static void assertKey(final byte [] expected, final byte [] got) { - if (!org.apache.hadoop.hbase.util.Bytes.equals(expected, got)) { + assertKey(expected, got, 0, got.length); + } + + public static void assertKey(final byte [] expected, final byte [] gotArray, + final int gotArrayOffset, final int gotArrayLength) { + if (!org.apache.hadoop.hbase.util.Bytes.equals(expected, 0, expected.length, + gotArray, gotArrayOffset, gotArrayLength)) { throw new AssertionError("Expected " + org.apache.hadoop.hbase.util.Bytes.toString(expected) + - " but got " + org.apache.hadoop.hbase.util.Bytes.toString(got)); + " but got " + + org.apache.hadoop.hbase.util.Bytes.toString(gotArray, gotArrayOffset, gotArrayLength)); } }