Index: hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java =================================================================== --- hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java (revision 1595551) +++ hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java (working copy) @@ -77,6 +77,12 @@ "ENCODE_ON_DISK"; public static final String DATA_BLOCK_ENCODING = "DATA_BLOCK_ENCODING"; + /** + * Key for the BLOCKCACHE attribute. + * A more exact name would be CACHE_DATA_ON_READ because this flag sets whether or not we + * cache DATA blocks. We always cache INDEX and BLOOM blocks; caching these blocks cannot be + * disabled. + */ public static final String BLOCKCACHE = "BLOCKCACHE"; public static final String CACHE_DATA_ON_WRITE = "CACHE_DATA_ON_WRITE"; public static final String CACHE_INDEX_ON_WRITE = "CACHE_INDEX_ON_WRITE"; @@ -792,7 +798,8 @@ } /** - * @return True if MapFile blocks should be cached. + * @return True if hfile DATA type blocks should be cached (You cannot disable caching of INDEX + * and BLOOM type blocks). */ public boolean isBlockCacheEnabled() { String value = getValue(BLOCKCACHE); @@ -802,7 +809,8 @@ } /** - * @param blockCacheEnabled True if MapFile blocks should be cached. + * @param blockCacheEnabled True if hfile DATA type blocks should be cached (We always cache + * INDEX and BLOOM blocks; you cannot turn this off). * @return this (for chained invocation) */ public HColumnDescriptor setBlockCacheEnabled(boolean blockCacheEnabled) { Index: hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java (revision 1595551) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java (working copy) @@ -161,7 +161,9 @@ /** * Whether blocks should be cached on read (default is on if there is a - * cache but this can be turned off on a per-family or per-request basis) + * cache but this can be turned off on a per-family or per-request basis). + * If off we will STILL cache meta blocks; i.e. INDEX and BLOOM types. + * This cannot be disabled. */ private boolean cacheDataOnRead; @@ -232,7 +234,8 @@ * Create a block cache configuration with the specified cache and * configuration parameters. * @param blockCache reference to block cache, null if completely disabled - * @param cacheDataOnRead whether data blocks should be cached on read + * @param cacheDataOnRead whether DATA blocks should be cached on read (we always cache INDEX + * blocks and BLOOM blocks; this cannot be disabled). * @param inMemory whether blocks should be flagged as in-memory * @param cacheDataOnWrite whether data blocks should be cached on write * @param cacheIndexesOnWrite whether index blocks should be cached on write @@ -283,7 +286,8 @@ } /** - * Returns whether the blocks of this HFile should be cached on read or not. + * Returns whether the DATA blocks of this HFile should be cached on read or not (we always + * cache the meta blocks, the INDEX and BLOOM blocks). * @return true if blocks should be cached on read, false if not */ public boolean shouldCacheDataOnRead() { Index: hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java (revision 1595551) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java (working copy) @@ -87,6 +87,12 @@ this.requestCachingCounts = initializeZeros(numPeriodsInWindow); } + @Override + public String toString() { + return "hitCount=" + this.hitCount + ", hitCachingCount=" + this.hitCachingCount + + ", missCount=" + this.missCount; + } + public void miss(boolean caching) { missCount.incrementAndGet(); if (caching) missCachingCount.incrementAndGet(); Index: hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java (revision 1595551) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java (working copy) @@ -16,11 +16,12 @@ */ package org.apache.hadoop.hbase.io.hfile; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.List; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; @@ -29,10 +30,10 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; -import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -43,11 +44,14 @@ * Make sure we always cache important block types, such as index blocks, as * long as we have a block cache, even though block caching might be disabled * for the column family. + * + *
TODO: This test writes a lot of data and only tests the most basic of metrics. Cache stats
+ * need to reveal more about what is being cached whether DATA or INDEX blocks and then we could
+ * do more verification in this test.
*/
@Category(MediumTests.class)
@RunWith(Parameterized.class)
public class TestForceCacheImportantBlocks {
-
private final HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
private static final String TABLE = "myTable";
@@ -68,6 +72,8 @@
Compression.Algorithm.GZ;
private static final BloomType BLOOM_TYPE = BloomType.ROW;
+ @SuppressWarnings("unused")
+ // Currently unused.
private final int hfileVersion;
private final boolean cfCacheEnabled;
@@ -76,7 +82,7 @@
// HFile versions
return Arrays.asList(new Object[][] {
new Object[] { new Integer(2), false },
- new Object[] { new Integer(2), true }
+ new Object[] { new Integer(3), true }
});
}
@@ -84,42 +90,42 @@
boolean cfCacheEnabled) {
this.hfileVersion = hfileVersion;
this.cfCacheEnabled = cfCacheEnabled;
- TEST_UTIL.getConfiguration().setInt(HFile.FORMAT_VERSION_KEY,
- hfileVersion);
+ TEST_UTIL.getConfiguration().setInt(HFile.FORMAT_VERSION_KEY, hfileVersion);
}
+ @Before
+ public void setup() {
+ // Make sure we make a new one each time.
+ CacheConfig.GLOBAL_BLOCK_CACHE_INSTANCE = null;
+ HFile.dataBlockReadCnt.set(0);
+ }
+
@Test
public void testCacheBlocks() throws IOException {
// Set index block size to be the same as normal block size.
- TEST_UTIL.getConfiguration().setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY,
- BLOCK_SIZE);
-
-
- HColumnDescriptor hcd =
- new HColumnDescriptor(Bytes.toBytes(CF))
- .setMaxVersions(MAX_VERSIONS)
- .setCompressionType(COMPRESSION_ALGORITHM)
- .setBloomFilterType(BLOOM_TYPE);
+ TEST_UTIL.getConfiguration().setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, BLOCK_SIZE);
+ HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes(CF)).setMaxVersions(MAX_VERSIONS).
+ setCompressionType(COMPRESSION_ALGORITHM).
+ setBloomFilterType(BLOOM_TYPE);
hcd.setBlocksize(BLOCK_SIZE);
hcd.setBlockCacheEnabled(cfCacheEnabled);
HRegion region = TEST_UTIL.createTestRegion(TABLE, hcd);
writeTestData(region);
-
- for (int i = 0; i < NUM_ROWS; ++i) {
- Get get = new Get(Bytes.toBytes("row" + i));
- region.get(get);
- }
-
- List