diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index 0b6ceef..df42b9a 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -248,7 +248,7 @@ public class HFile { protected FileSystem fs; protected Path path; protected FSDataOutputStream ostream; - protected CellComparator comparator = + protected CellComparator comparator = CellComparator.COMPARATOR; protected InetSocketAddress[] favoredNodes; private HFileContext fileContext; @@ -459,9 +459,9 @@ public class HFile { * Return the file context of the HFile this reader belongs to */ HFileContext getFileContext(); - + boolean isPrimaryReplicaReader(); - + void setPrimaryReplicaReader(boolean isPrimaryReplicaReader); boolean shouldIncludeMemstoreTS(); @@ -555,7 +555,7 @@ public class HFile { */ public static Reader createReader( FileSystem fs, Path path, CacheConfig cacheConf, Configuration conf) throws IOException { - Preconditions.checkNotNull(cacheConf, "Cannot create Reader with null CacheConf"); + //Preconditions.checkNotNull(cacheConf, "Cannot create Reader with null CacheConf"); FSDataInputStreamWrapper stream = new FSDataInputStreamWrapper(fs, path); return pickReaderVersion(path, stream, fs.getFileStatus(path).getLen(), cacheConf, stream.getHfs(), conf); @@ -655,82 +655,102 @@ public class HFile { return this; } + @Override public void clear() { this.map.clear(); } + @Override public Comparator comparator() { return map.comparator(); } + @Override public boolean containsKey(Object key) { return map.containsKey(key); } + @Override public boolean containsValue(Object value) { return map.containsValue(value); } + @Override public Set> entrySet() { return map.entrySet(); } + @Override public boolean equals(Object o) { return map.equals(o); } + @Override public byte[] firstKey() { return map.firstKey(); } + @Override public byte[] get(Object key) { return map.get(key); } + @Override public int hashCode() { return map.hashCode(); } + @Override public SortedMap headMap(byte[] toKey) { return this.map.headMap(toKey); } + @Override public boolean isEmpty() { return map.isEmpty(); } + @Override public Set keySet() { return map.keySet(); } + @Override public byte[] lastKey() { return map.lastKey(); } + @Override public byte[] put(byte[] key, byte[] value) { return this.map.put(key, value); } + @Override public void putAll(Map m) { this.map.putAll(m); } + @Override public byte[] remove(Object key) { return this.map.remove(key); } + @Override public int size() { return map.size(); } + @Override public SortedMap subMap(byte[] fromKey, byte[] toKey) { return this.map.subMap(fromKey, toKey); } + @Override public SortedMap tailMap(byte[] fromKey) { return this.map.tailMap(fromKey); } + @Override public Collection values() { return map.values(); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java index 4cf1bf2..9159e3f 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java @@ -244,8 +244,9 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { } // Prefetch file blocks upon open if requested - if (cacheConf.shouldPrefetchOnOpen()) { + if (cacheConf != null && cacheConf.shouldPrefetchOnOpen()) { PrefetchExecutor.request(path, new Runnable() { + @Override public void run() { long offset = 0; long end = 0; @@ -362,7 +363,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { @Override public void returnBlock(HFileBlock block) { - BlockCache blockCache = this.cacheConf.getBlockCache(); + BlockCache blockCache = cacheConf !=null ? this.cacheConf.getBlockCache(): null; if (blockCache != null && block != null) { BlockCacheKey cacheKey = new BlockCacheKey(this.getFileContext().getHFileName(), block.getOffset(), this.isPrimaryReplicaReader(), block.getBlockType()); @@ -430,6 +431,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { * @return the total heap size of data and meta block indexes in bytes. Does * not take into account non-root blocks of a multilevel data index. */ + @Override public long indexSize() { return (dataBlockIndexReader != null ? dataBlockIndexReader.heapSize() : 0) + ((metaBlockIndexReader != null) ? metaBlockIndexReader.heapSize() @@ -869,7 +871,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { return false; } - // The first key in the current block 'seekToBlock' is greater than the given + // The first key in the current block 'seekToBlock' is greater than the given // seekBefore key. We will go ahead by reading the next block that satisfies the // given key. Return the current block before reading the next one. reader.returnBlock(seekToBlock); @@ -983,7 +985,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { + keyPair.getSecond(), currKeyLen); } else { // Better to do a copy here instead of holding on to this BB so that - // we could release the blocks referring to this key. This key is specifically used + // we could release the blocks referring to this key. This key is specifically used // in HalfStoreFileReader to get the firstkey and lastkey by creating a new scanner // every time. So holding onto the BB (incase of DBB) is not advised here. byte[] key = new byte[currKeyLen]; @@ -1234,6 +1236,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { } } + @Override public Path getPath() { return path; } @@ -1271,10 +1274,12 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { protected boolean decodeMemstoreTS = false; + @Override public boolean isDecodeMemstoreTS() { return this.decodeMemstoreTS; } + @Override public boolean shouldIncludeMemstoreTS() { return includesMemstoreTS; } @@ -1287,7 +1292,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { boolean isCompaction, boolean updateCacheMetrics, BlockType expectedBlockType, DataBlockEncoding expectedDataBlockEncoding) throws IOException { // Check cache for block. If found return. - if (cacheConf.isBlockCacheEnabled()) { + if (cacheConf != null && cacheConf.isBlockCacheEnabled()) { BlockCache cache = cacheConf.getBlockCache(); HFileBlock cachedBlock = (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock, updateCacheMetrics); @@ -1380,8 +1385,9 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset, this.isPrimaryReplicaReader(), BlockType.META); - cacheBlock &= cacheConf.shouldCacheBlockOnRead(BlockType.META.getCategory()); - if (cacheConf.isBlockCacheEnabled()) { + cacheBlock &= cacheConf != null && + cacheConf.shouldCacheBlockOnRead(BlockType.META.getCategory()); + if (cacheConf != null && cacheConf.isBlockCacheEnabled()) { HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, true, BlockType.META, null); if (cachedBlock != null) { @@ -1397,6 +1403,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { unpack(hfileContext, fsBlockReader); // Cache the block + // cacheBlock == false if cacheConf == null if (cacheBlock) { cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock, cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1()); @@ -1436,7 +1443,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { try { while (true) { // Check cache for block. If found return. - if (cacheConf.shouldReadBlockFromCache(expectedBlockType)) { + if (cacheConf != null && cacheConf.shouldReadBlockFromCache(expectedBlockType)) { if (useLock) { lockEntry = offsetLock.getLockEntry(dataBlockOffset); } @@ -1487,7 +1494,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { BlockType.BlockCategory category = hfileBlock.getBlockType().getCategory(); // Cache the block if necessary - if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) { + if (cacheConf != null && cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) { cacheConf.getBlockCache().cacheBlock(cacheKey, cacheConf.shouldCacheCompressed(category) ? hfileBlock : unpacked, cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1()); @@ -1560,9 +1567,14 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { @Override public void close() throws IOException { - close(cacheConf.shouldEvictOnClose()); + if(cacheConf != null) { + close(cacheConf.shouldEvictOnClose()); + } else { + close(false); + } } + @Override public void close(boolean evictOnClose) throws IOException { PrefetchExecutor.cancel(path); if (evictOnClose && cacheConf.isBlockCacheEnabled()) { @@ -1575,11 +1587,13 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { fsBlockReader.closeStreams(); } + @Override public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) { return dataBlockEncoder.getEffectiveEncodingInCache(isCompaction); } /** For testing */ + @Override public HFileBlock.FSReader getUncachedBlockReader() { return fsBlockReader; } @@ -1607,6 +1621,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { return curBlock != null; } + @Override public void setNonSeekedState() { reset(); } @@ -1708,6 +1723,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { } } + @Override protected Cell getFirstKeyCellInBlock(HFileBlock curBlock) { return dataBlockEncoder.getFirstKeyCellInBlock(getEncodedBuffer(curBlock)); } @@ -1725,6 +1741,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { return seeker.seekToKeyInBlock(key, seekBefore); } + @Override public int compareKey(CellComparator comparator, Cell key) { return seeker.compareKey(comparator, key); } @@ -1771,6 +1788,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { * Returns false if block prefetching was requested for this file and has * not completed, true otherwise */ + @Override @VisibleForTesting public boolean prefetchComplete() { return PrefetchExecutor.isCompleted(path); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java index 73e580c..6fd652a 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java @@ -18,12 +18,18 @@ */ package org.apache.hadoop.hbase.io.hfile; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; -import java.util.Map; +import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -33,6 +39,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.ArrayBackedTag; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; @@ -42,21 +49,21 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.Tag; -import org.apache.hadoop.hbase.ArrayBackedTag; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.hfile.HFile.Reader; import org.apache.hadoop.hbase.io.hfile.HFile.Writer; import org.apache.hadoop.hbase.nio.ByteBuff; +import org.apache.hadoop.hbase.regionserver.StoreFileWriter; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.Writable; -import org.junit.*; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; -import static org.junit.Assert.*; - /** * test hfile features. */ @@ -66,6 +73,8 @@ public class TestHFile { @Rule public TestName testName = new TestName(); private static final Log LOG = LogFactory.getLog(TestHFile.class); + private static final int NUM_VALID_KEY_TYPES = KeyValue.Type.values().length - 2; + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static String ROOT_DIR = @@ -73,7 +82,6 @@ public class TestHFile { private final int minBlockSize = 512; private static String localFormatter = "%010d"; private static CacheConfig cacheConf = null; - private Map startingMetrics; private static Configuration conf ; private static FileSystem fs; @@ -82,7 +90,7 @@ public class TestHFile { conf = TEST_UTIL.getConfiguration(); fs = TEST_UTIL.getTestFileSystem(); } - + /** * Test empty HFile. * Test all features work reasonably when hfile is empty of entries. @@ -102,6 +110,12 @@ public class TestHFile { assertNull(r.getLastKey()); } + @Test + public void testReaderWithoutBlockCache() throws Exception { + Path path = writeStoreFile(); + readStoreFile(path); + } + /** * Create 0-length hfile and show that it fails */ @@ -450,7 +464,7 @@ public class TestHFile { mid = HFileWriterImpl.getMidpoint(CellComparator.COMPARATOR, left, right); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(left, mid) < 0); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(mid, right) < 0); - assertEquals(1, (int) mid.getRowLength()); + assertEquals(1, mid.getRowLength()); left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("a")); @@ -463,21 +477,21 @@ public class TestHFile { mid = HFileWriterImpl.getMidpoint(CellComparator.COMPARATOR, left, right); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(left, mid) < 0); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(mid, right) < 0); - assertEquals(2, (int) mid.getFamilyLength()); + assertEquals(2, mid.getFamilyLength()); left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaaa")); mid = HFileWriterImpl.getMidpoint(CellComparator.COMPARATOR, left, right); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(left, mid) < 0); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(mid, right) < 0); - assertEquals(2, (int) mid.getQualifierLength()); + assertEquals(2, mid.getQualifierLength()); left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("b")); mid = HFileWriterImpl.getMidpoint(CellComparator.COMPARATOR, left, right); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(left, mid) < 0); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(mid, right) <= 0); - assertEquals(1, (int) mid.getQualifierLength()); + assertEquals(1, mid.getQualifierLength()); // Assert that if meta comparator, it returns the right cell -- i.e. no // optimization done. @@ -564,5 +578,62 @@ public class TestHFile { 0, expectedArray.length); } + + private void readStoreFile(Path storeFilePath) throws Exception { + // Open the file (no cache) + HFile.Reader reader = HFile.createReader(fs, storeFilePath, null, conf); + long offset = 0; + while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { + HFileBlock block = reader.readBlock(offset, -1, false, true, false, true, null, null); + offset += block.getOnDiskSizeWithHeader(); + } + } + + private Path writeStoreFile() throws IOException { + Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile"); + HFileContext meta = new HFileContextBuilder() + .withBlockSize(64*1024) + .build(); + StoreFileWriter sfw = new StoreFileWriter.Builder(conf, cacheConf, fs) + .withOutputDir(storeFileParentDir) + .withComparator(CellComparator.COMPARATOR) + .withFileContext(meta) + .build(); + + final int rowLen = 32; + Random RNG = new Random(); + for (int i = 0; i < 1000; ++i) { + byte[] k = RandomKeyValueUtil.randomOrderedKey(RNG, i); + byte[] v = RandomKeyValueUtil.randomValue(RNG); + int cfLen = RNG.nextInt(k.length - rowLen + 1); + KeyValue kv = new KeyValue( + k, 0, rowLen, + k, rowLen, cfLen, + k, rowLen + cfLen, k.length - rowLen - cfLen, + RNG.nextLong(), + generateKeyType(RNG), + v, 0, v.length); + sfw.append(kv); + } + + sfw.close(); + return sfw.getPath(); + } + + public static KeyValue.Type generateKeyType(Random rand) { + if (rand.nextBoolean()) { + // Let's make half of KVs puts. + return KeyValue.Type.Put; + } else { + KeyValue.Type keyType = + KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)]; + if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum) + { + throw new RuntimeException("Generated an invalid key type: " + keyType + + ". " + "Probably the layout of KeyValue.Type has changed."); + } + return keyType; + } + } }