diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index e9fa05c..648c046 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -249,7 +249,7 @@ public class HFile { protected FileSystem fs; protected Path path; protected FSDataOutputStream ostream; - protected CellComparator comparator = + protected CellComparator comparator = CellComparator.COMPARATOR; protected InetSocketAddress[] favoredNodes; private HFileContext fileContext; @@ -460,9 +460,9 @@ public class HFile { * Return the file context of the HFile this reader belongs to */ HFileContext getFileContext(); - + boolean isPrimaryReplicaReader(); - + void setPrimaryReplicaReader(boolean isPrimaryReplicaReader); boolean shouldIncludeMemstoreTS(); @@ -547,6 +547,20 @@ public class HFile { } /** + * Creates reader w/o cache being involved + * @param fs filesystem + * @param path Path to file to read + * @return an active Reader instance + * @throws IOException Will throw a CorruptHFileException (DoNotRetryIOException subtype) if hfile is corrupt/invalid. + */ + public static Reader createReader( + FileSystem fs, Path path, Configuration conf) throws IOException { + FSDataInputStreamWrapper stream = new FSDataInputStreamWrapper(fs, path); + return pickReaderVersion(path, stream, fs.getFileStatus(path).getLen(), + null, stream.getHfs(), conf); + } + + /** * * @param fs filesystem * @param path Path to file to read @@ -556,7 +570,6 @@ public class HFile { */ public static Reader createReader( FileSystem fs, Path path, CacheConfig cacheConf, Configuration conf) throws IOException { - Preconditions.checkNotNull(cacheConf, "Cannot create Reader with null CacheConf"); FSDataInputStreamWrapper stream = new FSDataInputStreamWrapper(fs, path); return pickReaderVersion(path, stream, fs.getFileStatus(path).getLen(), cacheConf, stream.getHfs(), conf); @@ -656,82 +669,102 @@ public class HFile { return this; } + @Override public void clear() { this.map.clear(); } + @Override public Comparator comparator() { return map.comparator(); } + @Override public boolean containsKey(Object key) { return map.containsKey(key); } + @Override public boolean containsValue(Object value) { return map.containsValue(value); } + @Override public Set> entrySet() { return map.entrySet(); } + @Override public boolean equals(Object o) { return map.equals(o); } + @Override public byte[] firstKey() { return map.firstKey(); } + @Override public byte[] get(Object key) { return map.get(key); } + @Override public int hashCode() { return map.hashCode(); } + @Override public SortedMap headMap(byte[] toKey) { return this.map.headMap(toKey); } + @Override public boolean isEmpty() { return map.isEmpty(); } + @Override public Set keySet() { return map.keySet(); } + @Override public byte[] lastKey() { return map.lastKey(); } + @Override public byte[] put(byte[] key, byte[] value) { return this.map.put(key, value); } + @Override public void putAll(Map m) { this.map.putAll(m); } + @Override public byte[] remove(Object key) { return this.map.remove(key); } + @Override public int size() { return map.size(); } + @Override public SortedMap subMap(byte[] fromKey, byte[] toKey) { return this.map.subMap(fromKey, toKey); } + @Override public SortedMap tailMap(byte[] fromKey) { return this.map.tailMap(fromKey); } + @Override public Collection values() { return map.values(); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java index 239c63d..bab0669 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java @@ -245,8 +245,9 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { } // Prefetch file blocks upon open if requested - if (cacheConf.shouldPrefetchOnOpen()) { + if (cacheConf != null && cacheConf.shouldPrefetchOnOpen()) { PrefetchExecutor.request(path, new Runnable() { + @Override public void run() { long offset = 0; long end = 0; @@ -354,7 +355,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { @Override public void returnBlock(HFileBlock block) { - BlockCache blockCache = this.cacheConf.getBlockCache(); + BlockCache blockCache = cacheConf !=null ? this.cacheConf.getBlockCache(): null; if (blockCache != null && block != null) { BlockCacheKey cacheKey = new BlockCacheKey(this.getFileContext().getHFileName(), block.getOffset(), this.isPrimaryReplicaReader()); @@ -422,6 +423,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { * @return the total heap size of data and meta block indexes in bytes. Does * not take into account non-root blocks of a multilevel data index. */ + @Override public long indexSize() { return (dataBlockIndexReader != null ? dataBlockIndexReader.heapSize() : 0) + ((metaBlockIndexReader != null) ? metaBlockIndexReader.heapSize() @@ -856,7 +858,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { return false; } - // The first key in the current block 'seekToBlock' is greater than the given + // The first key in the current block 'seekToBlock' is greater than the given // seekBefore key. We will go ahead by reading the next block that satisfies the // given key. Return the current block before reading the next one. reader.returnBlock(seekToBlock); @@ -996,7 +998,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { + keyPair.getSecond(), currKeyLen); } else { // Better to do a copy here instead of holding on to this BB so that - // we could release the blocks referring to this key. This key is specifically used + // we could release the blocks referring to this key. This key is specifically used // in HalfStoreFileReader to get the firstkey and lastkey by creating a new scanner // every time. So holding onto the BB (incase of DBB) is not advised here. byte[] key = new byte[currKeyLen]; @@ -1280,6 +1282,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { } } + @Override public Path getPath() { return path; } @@ -1317,10 +1320,12 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { protected boolean decodeMemstoreTS = false; + @Override public boolean isDecodeMemstoreTS() { return this.decodeMemstoreTS; } + @Override public boolean shouldIncludeMemstoreTS() { return includesMemstoreTS; } @@ -1333,7 +1338,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { boolean isCompaction, boolean updateCacheMetrics, BlockType expectedBlockType, DataBlockEncoding expectedDataBlockEncoding) throws IOException { // Check cache for block. If found return. - if (cacheConf.isBlockCacheEnabled()) { + if (cacheConf != null && cacheConf.isBlockCacheEnabled()) { BlockCache cache = cacheConf.getBlockCache(); HFileBlock cachedBlock = (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock, updateCacheMetrics); @@ -1426,8 +1431,8 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset, this.isPrimaryReplicaReader()); - cacheBlock &= cacheConf.shouldCacheDataOnRead(); - if (cacheConf.isBlockCacheEnabled()) { + cacheBlock &= cacheConf != null && cacheConf.shouldCacheDataOnRead(); + if (cacheConf != null && cacheConf.isBlockCacheEnabled()) { HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, true, BlockType.META, null); if (cachedBlock != null) { @@ -1443,6 +1448,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { blockSize, -1, true).unpack(hfileContext, fsBlockReader); // Cache the block + // cacheBlock == false if cacheConf == null if (cacheBlock) { cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock, cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1()); @@ -1482,7 +1488,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { try { while (true) { // Check cache for block. If found return. - if (cacheConf.shouldReadBlockFromCache(expectedBlockType)) { + if (cacheConf != null && cacheConf.shouldReadBlockFromCache(expectedBlockType)) { if (useLock) { lockEntry = offsetLock.getLockEntry(dataBlockOffset); } @@ -1530,7 +1536,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { BlockType.BlockCategory category = hfileBlock.getBlockType().getCategory(); // Cache the block if necessary - if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) { + if (cacheConf != null && cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) { cacheConf.getBlockCache().cacheBlock(cacheKey, cacheConf.shouldCacheCompressed(category) ? hfileBlock : unpacked, cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1()); @@ -1603,9 +1609,14 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { @Override public void close() throws IOException { - close(cacheConf.shouldEvictOnClose()); + if(cacheConf != null) { + close(cacheConf.shouldEvictOnClose()); + } else { + close(false); + } } + @Override public void close(boolean evictOnClose) throws IOException { PrefetchExecutor.cancel(path); if (evictOnClose && cacheConf.isBlockCacheEnabled()) { @@ -1618,11 +1629,13 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { fsBlockReader.closeStreams(); } + @Override public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) { return dataBlockEncoder.getEffectiveEncodingInCache(isCompaction); } /** For testing */ + @Override public HFileBlock.FSReader getUncachedBlockReader() { return fsBlockReader; } @@ -1650,6 +1663,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { return curBlock != null; } + @Override public void setNonSeekedState() { reset(); } @@ -1751,6 +1765,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { } } + @Override protected Cell getFirstKeyCellInBlock(HFileBlock curBlock) { return dataBlockEncoder.getFirstKeyCellInBlock(getEncodedBuffer(curBlock)); } @@ -1768,6 +1783,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { return seeker.seekToKeyInBlock(key, seekBefore); } + @Override public int compareKey(CellComparator comparator, Cell key) { return seeker.compareKey(comparator, key); } @@ -1814,6 +1830,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { * Returns false if block prefetching was requested for this file and has * not completed, true otherwise */ + @Override @VisibleForTesting public boolean prefetchComplete() { return PrefetchExecutor.isCompleted(path); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java index 66fb49c..9bdb259 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java @@ -18,12 +18,18 @@ */ package org.apache.hadoop.hbase.io.hfile; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; -import java.util.Map; +import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -33,6 +39,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.ArrayBackedTag; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; @@ -42,21 +49,21 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.Tag; -import org.apache.hadoop.hbase.ArrayBackedTag; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.hfile.HFile.Reader; import org.apache.hadoop.hbase.io.hfile.HFile.Writer; import org.apache.hadoop.hbase.nio.ByteBuff; +import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.Writable; -import org.junit.*; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; -import static org.junit.Assert.*; - /** * test hfile features. */ @@ -66,6 +73,8 @@ public class TestHFile { @Rule public TestName testName = new TestName(); private static final Log LOG = LogFactory.getLog(TestHFile.class); + private static final int NUM_VALID_KEY_TYPES = KeyValue.Type.values().length - 2; + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static String ROOT_DIR = @@ -73,7 +82,6 @@ public class TestHFile { private final int minBlockSize = 512; private static String localFormatter = "%010d"; private static CacheConfig cacheConf = null; - private Map startingMetrics; private static Configuration conf ; private static FileSystem fs; @@ -82,7 +90,7 @@ public class TestHFile { conf = TEST_UTIL.getConfiguration(); fs = TEST_UTIL.getTestFileSystem(); } - + /** * Test empty HFile. * Test all features work reasonably when hfile is empty of entries. @@ -102,6 +110,12 @@ public class TestHFile { assertNull(r.getLastKey()); } + @Test + public void testReaderWithoutBlockCache() throws Exception { + Path path = writeStoreFile(); + readStoreFile(path); + } + /** * Create 0-length hfile and show that it fails */ @@ -430,7 +444,7 @@ public class TestHFile { mid = HFileWriterImpl.getMidpoint(CellComparator.COMPARATOR, left, right); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(left, mid) < 0); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(mid, right) < 0); - assertEquals(1, (int) mid.getRowLength()); + assertEquals(1, mid.getRowLength()); left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("a")); @@ -443,21 +457,21 @@ public class TestHFile { mid = HFileWriterImpl.getMidpoint(CellComparator.COMPARATOR, left, right); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(left, mid) < 0); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(mid, right) < 0); - assertEquals(2, (int) mid.getFamilyLength()); + assertEquals(2, mid.getFamilyLength()); left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaaa")); mid = HFileWriterImpl.getMidpoint(CellComparator.COMPARATOR, left, right); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(left, mid) < 0); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(mid, right) < 0); - assertEquals(2, (int) mid.getQualifierLength()); + assertEquals(2, mid.getQualifierLength()); left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("b")); mid = HFileWriterImpl.getMidpoint(CellComparator.COMPARATOR, left, right); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(left, mid) < 0); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(mid, right) <= 0); - assertEquals(1, (int) mid.getQualifierLength()); + assertEquals(1, mid.getQualifierLength()); // Assert that if meta comparator, it returns the right cell -- i.e. no // optimization done. @@ -544,5 +558,62 @@ public class TestHFile { 0, expectedArray.length); } + + private void readStoreFile(Path storeFilePath) throws Exception { + // Open the file (no cache) + HFile.Reader reader = HFile.createReader(fs, storeFilePath, conf); + long offset = 0; + while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { + HFileBlock block = reader.readBlock(offset, -1, false, true, false, true, null, null); + offset += block.getOnDiskSizeWithHeader(); + } + } + + private Path writeStoreFile() throws IOException { + Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile"); + HFileContext meta = new HFileContextBuilder() + .withBlockSize(64*1024) + .build(); + StoreFile.Writer sfw = new StoreFile.WriterBuilder(conf, cacheConf, fs) + .withOutputDir(storeFileParentDir) + .withComparator(CellComparator.COMPARATOR) + .withFileContext(meta) + .build(); + + final int rowLen = 32; + Random RNG = new Random(); + for (int i = 0; i < 1000; ++i) { + byte[] k = RandomKeyValueUtil.randomOrderedKey(RNG, i); + byte[] v = RandomKeyValueUtil.randomValue(RNG); + int cfLen = RNG.nextInt(k.length - rowLen + 1); + KeyValue kv = new KeyValue( + k, 0, rowLen, + k, rowLen, cfLen, + k, rowLen + cfLen, k.length - rowLen - cfLen, + RNG.nextLong(), + generateKeyType(RNG), + v, 0, v.length); + sfw.append(kv); + } + + sfw.close(); + return sfw.getPath(); + } + + public static KeyValue.Type generateKeyType(Random rand) { + if (rand.nextBoolean()) { + // Let's make half of KVs puts. + return KeyValue.Type.Put; + } else { + KeyValue.Type keyType = + KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)]; + if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum) + { + throw new RuntimeException("Generated an invalid key type: " + keyType + + ". " + "Probably the layout of KeyValue.Type has changed."); + } + return keyType; + } + } }