diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
index b696f41..d89bd05 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
@@ -684,4 +684,4 @@ public final class CellUtil {
}
return commonPrefix;
}
-}
+}
\ No newline at end of file
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
index b367a0d..f963155 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -202,29 +202,6 @@ public class HFile {
void append(Cell cell) throws IOException;
- /**
- * @param key
- * @param value
- * @throws IOException
- * @Deprecated Use {@link #append(Cell)}
- */
- // This method makes no sense anymore. What is 'key'? Row? How can you have a 'row' w/o
- // a family, etc.
- @Deprecated
- void append(byte[] key, byte[] value) throws IOException;
-
- // This method makes no sense anymore. What is 'key'? Row? How can you have a 'row' w/o
- // a family, etc.
- /**
- * @param key
- * @param value
- * @param tag
- * @throws IOException
- * @Deprecated Use {@link #append(Cell)}
- */
- @Deprecated
- void append (byte[] key, byte[] value, byte[] tag) throws IOException;
-
/** @return the path to this {@link HFile} */
Path getPath();
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java
index b951fab..4be14b5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java
@@ -60,7 +60,7 @@ public interface HFileScanner {
@Deprecated
int seekTo(byte[] key, int offset, int length) throws IOException;
- int seekTo(Cell kv) throws IOException;
+ int seekTo(Cell c) throws IOException;
/**
* Reseek to or just before the passed key. Similar to seekTo
* except that this can be called even if the scanner is not at the beginning
@@ -86,7 +86,7 @@ public interface HFileScanner {
@Deprecated
int reseekTo(byte[] key, int offset, int length) throws IOException;
- int reseekTo(Cell kv) throws IOException;
+ int reseekTo(Cell c) throws IOException;
/**
* Consider the key stream of all the keys in the file,
* k[0] .. k[n], where there are n keys in the file.
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
index ee0e303..49211c8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.KVComparator;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
@@ -244,16 +243,22 @@ public class HFileWriterV2 extends AbstractHFileWriter {
* Add key/value to file. Keys must be added in an order that agrees with the
* Comparator passed on construction.
*
- * @param cell
- * Cell to add. Cannot be empty nor null.
+ * @param cell Cell to add. Cannot be empty nor null.
* @throws IOException
*/
@Override
public void append(final Cell cell) throws IOException {
+ // TODO:
+ // + Need to get a key from Cell to add to the index, to use as first key in block.
+ // It must work with passed in KVComparator.
+ // + For index, always needs to be in order? Always needs to be in order.
+ // Q: Can DBE destroy order?
+ // Should DBE be making the index key? It won't work with KVComparator if it does.
int klength = KeyValueUtil.keyLength(cell);
byte[] value = cell.getValueArray();
int voffset = cell.getValueOffset();
int vlength = cell.getValueLength();
+ // checkKey uses comparator to check we are writing in order.
boolean dupKey = checkKey(cell);
checkValue(value, voffset, vlength);
if (!dupKey) {
@@ -270,7 +275,7 @@ public class HFileWriterV2 extends AbstractHFileWriter {
// Are we the first key in this block?
if (firstKeyInBlock == null) {
- // Copy the key.
+ // Copy the key for use as first key in block. It is put into file index.
firstKeyInBlock = new byte[klength];
KeyValueUtil.appendKeyTo(cell, firstKeyInBlock, 0);
}
@@ -281,28 +286,6 @@ public class HFileWriterV2 extends AbstractHFileWriter {
this.maxMemstoreTS = Math.max(this.maxMemstoreTS, cell.getSequenceId());
}
- /**
- * Add key/value to file. Keys must be added in an order that agrees with the
- * Comparator passed on construction.
- *
- * @param key
- * Key to add. Cannot be empty nor null.
- * @param value
- * Value to add. Cannot be empty nor null.
- * @throws IOException
- */
- @Override
- public void append(final byte[] key, final byte[] value) throws IOException {
- int kvlen = (int) KeyValue.getKeyValueDataStructureSize(key.length, value.length, 0);
- byte[] b = new byte[kvlen];
- int pos = 0;
- pos = Bytes.putInt(b, pos, key.length);
- pos = Bytes.putInt(b, pos, value.length);
- pos = Bytes.putBytes(b, pos, key, 0, key.length);
- Bytes.putBytes(b, pos, value, 0, value.length);
- append(new KeyValue(b, 0, kvlen));
- }
-
@Override
public void close() throws IOException {
if (outputStream == null) {
@@ -426,11 +409,6 @@ public class HFileWriterV2 extends AbstractHFileWriter {
});
}
- @Override
- public void append(byte[] key, byte[] value, byte[] tag) throws IOException {
- throw new UnsupportedOperationException("KV tags are supported only from HFile V3");
- }
-
protected int getMajorVersion() {
return 2;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV3.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV3.java
index a0a2372..9afe372 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV3.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV3.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.KVComparator;
import org.apache.hadoop.hbase.io.crypto.Encryption;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
@@ -91,47 +90,6 @@ public class HFileWriterV3 extends HFileWriterV2 {
this.maxTagsLength = tagsLength;
}
}
-
- /**
- * Add key/value to file. Keys must be added in an order that agrees with the
- * Comparator passed on construction.
- * @param key
- * Key to add. Cannot be empty nor null.
- * @param value
- * Value to add. Cannot be empty nor null.
- * @throws IOException
- */
- @Override
- public void append(final byte[] key, final byte[] value) throws IOException {
- append(key, value, HConstants.EMPTY_BYTE_ARRAY);
- }
-
- /**
- * Add key/value to file. Keys must be added in an order that agrees with the
- * Comparator passed on construction.
- * @param key
- * Key to add. Cannot be empty nor null.
- * @param value
- * Value to add. Cannot be empty nor null.
- * @param tag
- * Tag t add. Cannot be empty or null.
- * @throws IOException
- */
- @Override
- public void append(final byte[] key, final byte[] value, byte[] tag) throws IOException {
- int kvlen = (int) KeyValue.getKeyValueDataStructureSize(key.length, value.length, tag.length);
- byte[] b = new byte[kvlen];
- int pos = 0;
- pos = Bytes.putInt(b, pos, key.length);
- pos = Bytes.putInt(b, pos, value.length);
- pos = Bytes.putBytes(b, pos, key, 0, key.length);
- pos = Bytes.putBytes(b, pos, value, 0, value.length);
- if (tag.length > 0) {
- pos = Bytes.putAsShort(b, pos, tag.length);
- Bytes.putBytes(b, pos, tag, 0, tag.length);
- }
- append(new KeyValue(b, 0, kvlen));
- }
protected void finishFileInfo() throws IOException {
super.finishFileInfo();
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
index a1dca6f..7f2ffee 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
@@ -38,12 +38,9 @@ import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.util.Bytes;
/**
- *
* This class runs performance benchmarks for {@link HFile}. - *
*/ public class HFilePerformanceEvaluation { - private static final int ROW_LENGTH = 10; private static final int ROW_COUNT = 1000000; private static final int RFILE_BLOCKSIZE = 8 * 1024; @@ -61,6 +58,30 @@ public class HFilePerformanceEvaluation { return w; } + static Cell createCell(final int i) { + return createCell(i, HConstants.EMPTY_BYTE_ARRAY); + } + + /** + * HFile is Cell-based. It used to be byte arrays. Doing this test, pass Cells. All Cells + * intentionally have same coordinates in all fields but row. + * @param i Integer to format as a row Key. + * @param value Value to use + * @return Created Cell. + */ + static Cell createCell(final int i, final byte [] value) { + return createCell(format(i), value); + } + + static Cell createCell(final byte [] keyRow) { + return createCell(keyRow, HConstants.EMPTY_BYTE_ARRAY); + } + + static Cell createCell(final byte [] keyRow, final byte [] value) { + return CellUtil.createCell(keyRow, HConstants.CATALOG_FAMILY, HConstants.EMPTY_BYTE_ARRAY, + HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum.getCode(), value); + } + private void runBenchmarks() throws Exception { final Configuration conf = new Configuration(); final FileSystem fs = FileSystem.get(conf); @@ -200,10 +221,7 @@ public class HFilePerformanceEvaluation { @Override void doRow(int i) throws Exception { - Cell c = CellUtil.createCell(format(i), HConstants.CATALOG_FAMILY, - HConstants.CATALOG_FAMILY, HConstants.LATEST_TIMESTAMP, KeyValue.Type.Minimum.getCode(), - generateValue()); - writer.append(c); + writer.append(createCell(i, generateValue())); } private byte[] generateValue() { @@ -263,10 +281,10 @@ public class HFilePerformanceEvaluation { @Override void doRow(int i) throws Exception { if (this.scanner.next()) { - ByteBuffer k = this.scanner.getKey(); - PerformanceEvaluationCommons.assertKey(format(i + 1), k); - ByteBuffer v = scanner.getValue(); - PerformanceEvaluationCommons.assertValueSize(v.limit(), ROW_LENGTH); + // TODO: Fix. Make Scanner do Cells. + Cell c = this.scanner.getKeyValue(); + PerformanceEvaluationCommons.assertKey(format(i + 1), c); + PerformanceEvaluationCommons.assertValueSize(c.getValueLength(), ROW_LENGTH); } } @@ -290,14 +308,14 @@ public class HFilePerformanceEvaluation { void doRow(int i) throws Exception { HFileScanner scanner = this.reader.getScanner(false, true); byte [] b = getRandomRow(); - if (scanner.seekTo(b) < 0) { + if (scanner.seekTo(createCell(b)) < 0) { LOG.info("Not able to seekTo " + new String(b)); return; } - ByteBuffer k = scanner.getKey(); - PerformanceEvaluationCommons.assertKey(b, k); - ByteBuffer v = scanner.getValue(); - PerformanceEvaluationCommons.assertValueSize(v.limit(), ROW_LENGTH); + // TODO: Fix scanner so it does Cells + Cell c = scanner.getKeyValue(); + PerformanceEvaluationCommons.assertKey(b, c); + PerformanceEvaluationCommons.assertValueSize(c.getValueLength(), ROW_LENGTH); } private byte [] getRandomRow() { @@ -317,20 +335,24 @@ public class HFilePerformanceEvaluation { void doRow(int i) throws Exception { HFileScanner scanner = this.reader.getScanner(false, false); byte [] b = getRandomRow(); - if (scanner.seekTo(b) != 0) { + // System.out.println("Random row: " + new String(b)); + Cell c = createCell(b); + if (scanner.seekTo(c) != 0) { LOG.info("Nonexistent row: " + new String(b)); return; } - ByteBuffer k = scanner.getKey(); - PerformanceEvaluationCommons.assertKey(b, k); - // System.out.println("Found row: " + new String(b)); + // TODO: HFileScanner doesn't do Cells yet. Temporary fix. + c = scanner.getKeyValue(); + // System.out.println("Found row: " + + // new String(c.getRowArray(), c.getRowOffset(), c.getRowLength())); + PerformanceEvaluationCommons.assertKey(b, c); for (int ii = 0; ii < 30; ii++) { if (!scanner.next()) { LOG.info("NOTHING FOLLOWS"); return; } - ByteBuffer v = scanner.getValue(); - PerformanceEvaluationCommons.assertValueSize(v.limit(), ROW_LENGTH); + c = scanner.getKeyValue(); + PerformanceEvaluationCommons.assertValueSize(c.getValueLength(), ROW_LENGTH); } } @@ -352,14 +374,14 @@ public class HFilePerformanceEvaluation { void doRow(int i) throws Exception { HFileScanner scanner = this.reader.getScanner(false, true); byte[] gaussianRandomRowBytes = getGaussianRandomRowBytes(); - scanner.seekTo(gaussianRandomRowBytes); + scanner.seekTo(createCell(gaussianRandomRowBytes)); for (int ii = 0; ii < 30; ii++) { if (!scanner.next()) { LOG.info("NOTHING FOLLOWS"); return; } - scanner.getKey(); - scanner.getValue(); + // TODO: Fix. Make scanner do Cells. + scanner.getKeyValue(); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java index 6ac9865..4b0aa58 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java @@ -45,11 +45,22 @@ public class PerformanceEvaluationCommons { assertKey(expected, b); } + public static void assertKey(final byte [] expected, final Cell c) { + assertKey(expected, c.getRowArray(), c.getRowOffset(), c.getRowLength()); + } + public static void assertKey(final byte [] expected, final byte [] got) { - if (!org.apache.hadoop.hbase.util.Bytes.equals(expected, got)) { + assertKey(expected, got, 0, got.length); + } + + public static void assertKey(final byte [] expected, final byte [] gotArray, + final int gotArrayOffset, final int gotArrayLength) { + if (!org.apache.hadoop.hbase.util.Bytes.equals(expected, 0, expected.length, + gotArray, gotArrayOffset, gotArrayLength)) { throw new AssertionError("Expected " + org.apache.hadoop.hbase.util.Bytes.toString(expected) + - " but got " + org.apache.hadoop.hbase.util.Bytes.toString(got)); + " but got " + + org.apache.hadoop.hbase.util.Bytes.toString(gotArray, gotArrayOffset, gotArrayLength)); } }