From e22d5b73416a1ddfc448525dc181b751cd0e265e Mon Sep 17 00:00:00 2001 From: chenheng Date: Thu, 5 Nov 2015 17:58:33 +0800 Subject: [PATCH] HBASE-14189 CF Level BC setting should override global one --- .../org/apache/hadoop/hbase/HColumnDescriptor.java | 73 +++++++++--- .../apache/hadoop/hbase/TestHColumnDescriptor.java | 37 ++++++ .../apache/hadoop/hbase/io/hfile/CacheConfig.java | 124 ++++++++++++--------- .../hadoop/hbase/io/hfile/TestCacheConfig.java | 13 +++ .../hadoop/hbase/io/hfile/TestCacheOnWrite.java | 2 +- 5 files changed, 180 insertions(+), 69 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java index 4daf6ef..a2f113f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java @@ -79,10 +79,12 @@ public class HColumnDescriptor implements Comparable { * disabled. */ public static final String BLOCKCACHE = "BLOCKCACHE"; + public static final String CACHE_DATA_ON_READ = "CACHE_DATA_ON_READ"; public static final String CACHE_DATA_ON_WRITE = "CACHE_DATA_ON_WRITE"; public static final String CACHE_INDEX_ON_WRITE = "CACHE_INDEX_ON_WRITE"; public static final String CACHE_BLOOMS_ON_WRITE = "CACHE_BLOOMS_ON_WRITE"; public static final String EVICT_BLOCKS_ON_CLOSE = "EVICT_BLOCKS_ON_CLOSE"; + public static final String CACHE_DATA_BLOCKS_COMPRESSED = "CACHE_DATA_BLOCKS_COMPRESSED"; /** * Key for cache data into L1 if cache is set up with more than one tier. * To set in the shell, do something like this: @@ -133,6 +135,12 @@ public class HColumnDescriptor implements Comparable { public static final String DFS_REPLICATION = "DFS_REPLICATION"; public static final short DEFAULT_DFS_REPLICATION = 0; + public enum DefaultType { + TRUE, + FALSE, + NONE + } + /** * Default compression type. */ @@ -182,24 +190,27 @@ public class HColumnDescriptor implements Comparable { */ public static final boolean DEFAULT_BLOCKCACHE = true; + public static final DefaultType DEFAULT_CACHE_DATA_ON_READ = DefaultType.NONE; /** * Default setting for whether to cache data blocks on write if block caching * is enabled. */ - public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false; + public static final DefaultType DEFAULT_CACHE_DATA_ON_WRITE = DefaultType.NONE; /** * Default setting for whether to cache data blocks in L1 tier. Only makes sense if more than * one tier in operations: i.e. if we have an L1 and a L2. This will be the cases if we are * using BucketCache. */ - public static final boolean DEFAULT_CACHE_DATA_IN_L1 = false; + public static final DefaultType DEFAULT_CACHE_DATA_IN_L1 = DefaultType.NONE; /** * Default setting for whether to cache index blocks on write if block * caching is enabled. */ - public static final boolean DEFAULT_CACHE_INDEX_ON_WRITE = false; + public static final DefaultType DEFAULT_CACHE_INDEX_ON_WRITE = DefaultType.NONE; + + public static final DefaultType DEFAULT_CACHE_DATA_BLOCKS_COMPRESSED = DefaultType.NONE; /** * Default size of blocks in files stored to the filesytem (hfiles). @@ -215,7 +226,7 @@ public class HColumnDescriptor implements Comparable { * Default setting for whether to cache bloom filter blocks on write if block * caching is enabled. */ - public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false; + public static final DefaultType DEFAULT_CACHE_BLOOMS_ON_WRITE = DefaultType.NONE; /** * Default time to live of cell contents. @@ -231,7 +242,7 @@ public class HColumnDescriptor implements Comparable { * Default setting for whether to evict cached blocks from the blockcache on * close. */ - public static final boolean DEFAULT_EVICT_BLOCKS_ON_CLOSE = false; + public static final DefaultType DEFAULT_EVICT_BLOCKS_ON_CLOSE = DefaultType.NONE; /** * Default compress tags along with any type of DataBlockEncoding. @@ -241,7 +252,7 @@ public class HColumnDescriptor implements Comparable { /* * Default setting for whether to prefetch blocks into the blockcache on open. */ - public static final boolean DEFAULT_PREFETCH_BLOCKS_ON_OPEN = false; + public static final DefaultType DEFAULT_PREFETCH_BLOCKS_ON_OPEN = DefaultType.NONE; private final static Map DEFAULT_VALUES = new HashMap(); @@ -260,12 +271,15 @@ public class HColumnDescriptor implements Comparable { DEFAULT_VALUES.put(BLOCKCACHE, String.valueOf(DEFAULT_BLOCKCACHE)); DEFAULT_VALUES.put(KEEP_DELETED_CELLS, String.valueOf(DEFAULT_KEEP_DELETED)); DEFAULT_VALUES.put(DATA_BLOCK_ENCODING, String.valueOf(DEFAULT_DATA_BLOCK_ENCODING)); + DEFAULT_VALUES.put(CACHE_DATA_ON_READ, String.valueOf(DEFAULT_CACHE_DATA_ON_READ)); DEFAULT_VALUES.put(CACHE_DATA_ON_WRITE, String.valueOf(DEFAULT_CACHE_DATA_ON_WRITE)); DEFAULT_VALUES.put(CACHE_DATA_IN_L1, String.valueOf(DEFAULT_CACHE_DATA_IN_L1)); DEFAULT_VALUES.put(CACHE_INDEX_ON_WRITE, String.valueOf(DEFAULT_CACHE_INDEX_ON_WRITE)); DEFAULT_VALUES.put(CACHE_BLOOMS_ON_WRITE, String.valueOf(DEFAULT_CACHE_BLOOMS_ON_WRITE)); DEFAULT_VALUES.put(EVICT_BLOCKS_ON_CLOSE, String.valueOf(DEFAULT_EVICT_BLOCKS_ON_CLOSE)); DEFAULT_VALUES.put(PREFETCH_BLOCKS_ON_OPEN, String.valueOf(DEFAULT_PREFETCH_BLOCKS_ON_OPEN)); + DEFAULT_VALUES.put(CACHE_DATA_BLOCKS_COMPRESSED, + String.valueOf(DEFAULT_CACHE_DATA_BLOCKS_COMPRESSED)); for (String s : DEFAULT_VALUES.keySet()) { RESERVED_KEYWORDS.add(new Bytes(Bytes.toBytes(s))); } @@ -805,10 +819,17 @@ public class HColumnDescriptor implements Comparable { /** * @return true if we should cache data blocks on write */ - public boolean isCacheDataOnWrite() { - return setAndGetBoolean(CACHE_DATA_ON_WRITE, DEFAULT_CACHE_DATA_ON_WRITE); + public DefaultType isCacheDataOnWrite() { + return setAndGetDefaultType(CACHE_DATA_ON_WRITE, DEFAULT_CACHE_DATA_ON_WRITE); + } + + public DefaultType isCacheDataOnRead() { + return setAndGetDefaultType(CACHE_DATA_ON_READ, DEFAULT_CACHE_DATA_ON_READ); } + public DefaultType isCacheDataBlockCompressed() { + return setAndGetDefaultType(CACHE_DATA_BLOCKS_COMPRESSED, DEFAULT_CACHE_DATA_BLOCKS_COMPRESSED); + } /** * @param value true if we should cache data blocks on write * @return this (for chained invocation) @@ -817,12 +838,20 @@ public class HColumnDescriptor implements Comparable { return setValue(CACHE_DATA_ON_WRITE, Boolean.toString(value)); } + public HColumnDescriptor setCacheDataOnRead(boolean value) { + return setValue(CACHE_DATA_ON_READ, Boolean.toString(value)); + } + + public HColumnDescriptor setCacheDataBlockCompressed(boolean value) { + return setValue(CACHE_DATA_BLOCKS_COMPRESSED, Boolean.toString(value)); + } + /** * @return true if we should cache data blocks in the L1 cache (if block cache deploy has more * than one tier; e.g. we are using CombinedBlockCache). */ - public boolean isCacheDataInL1() { - return setAndGetBoolean(CACHE_DATA_IN_L1, DEFAULT_CACHE_DATA_IN_L1); + public DefaultType isCacheDataInL1() { + return setAndGetDefaultType(CACHE_DATA_IN_L1, DEFAULT_CACHE_DATA_IN_L1); } /** @@ -842,11 +871,19 @@ public class HColumnDescriptor implements Comparable { return defaultSetting; } + private DefaultType setAndGetDefaultType(final String key, final DefaultType defaultType) { + String value = getValue(key); + if (value != null) { + return DefaultType.valueOf(value.toUpperCase()); + } + return defaultType; + } + /** * @return true if we should cache index blocks on write */ - public boolean isCacheIndexesOnWrite() { - return setAndGetBoolean(CACHE_INDEX_ON_WRITE, DEFAULT_CACHE_INDEX_ON_WRITE); + public DefaultType isCacheIndexesOnWrite() { + return setAndGetDefaultType(CACHE_INDEX_ON_WRITE, DEFAULT_CACHE_INDEX_ON_WRITE); } /** @@ -860,8 +897,8 @@ public class HColumnDescriptor implements Comparable { /** * @return true if we should cache bloomfilter blocks on write */ - public boolean isCacheBloomsOnWrite() { - return setAndGetBoolean(CACHE_BLOOMS_ON_WRITE, DEFAULT_CACHE_BLOOMS_ON_WRITE); + public DefaultType isCacheBloomsOnWrite() { + return setAndGetDefaultType(CACHE_BLOOMS_ON_WRITE, DEFAULT_CACHE_BLOOMS_ON_WRITE); } /** @@ -875,8 +912,8 @@ public class HColumnDescriptor implements Comparable { /** * @return true if we should evict cached blocks from the blockcache on close */ - public boolean isEvictBlocksOnClose() { - return setAndGetBoolean(EVICT_BLOCKS_ON_CLOSE, DEFAULT_EVICT_BLOCKS_ON_CLOSE); + public DefaultType isEvictBlocksOnClose() { + return setAndGetDefaultType(EVICT_BLOCKS_ON_CLOSE, DEFAULT_EVICT_BLOCKS_ON_CLOSE); } /** @@ -891,8 +928,8 @@ public class HColumnDescriptor implements Comparable { /** * @return true if we should prefetch blocks into the blockcache on open */ - public boolean isPrefetchBlocksOnOpen() { - return setAndGetBoolean(PREFETCH_BLOCKS_ON_OPEN, DEFAULT_PREFETCH_BLOCKS_ON_OPEN); + public DefaultType isPrefetchBlocksOnOpen() { + return setAndGetDefaultType(PREFETCH_BLOCKS_ON_OPEN, DEFAULT_PREFETCH_BLOCKS_ON_OPEN); } /** diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptor.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptor.java index c53fff2..fb3bd99 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptor.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptor.java @@ -83,6 +83,43 @@ public class TestHColumnDescriptor { assertEquals(hcd.isMobEnabled(), deserializedHcd.isMobEnabled()); assertEquals(hcd.getMobThreshold(), deserializedHcd.getMobThreshold()); assertEquals(v, deserializedHcd.getDFSReplication()); + + assertEquals(HColumnDescriptor.DefaultType.NONE, hcd.isCacheDataOnWrite()); + hcd.setCacheDataOnWrite(true); + assertEquals(HColumnDescriptor.DefaultType.TRUE, hcd.isCacheDataOnWrite()); + } + + @Test + public void testDefaultType() { + HColumnDescriptor hcd = new HColumnDescriptor("testDefaultType"); + assertEquals(HColumnDescriptor.DefaultType.NONE, hcd.isCacheDataOnWrite()); + hcd.setCacheDataOnWrite(true); + assertEquals(HColumnDescriptor.DefaultType.TRUE, hcd.isCacheDataOnWrite()); + + assertEquals(HColumnDescriptor.DefaultType.NONE, hcd.isCacheDataOnRead()); + hcd.setCacheDataOnRead(false); + assertEquals(HColumnDescriptor.DefaultType.FALSE, hcd.isCacheDataOnRead()); + + assertEquals(HColumnDescriptor.DefaultType.NONE, hcd.isCacheBloomsOnWrite()); + hcd.setCacheBloomsOnWrite(false); + assertEquals(HColumnDescriptor.DefaultType.FALSE, hcd.isCacheBloomsOnWrite()); + + assertEquals(HColumnDescriptor.DefaultType.NONE, hcd.isCacheDataInL1()); + hcd.setCacheDataInL1(true); + assertEquals(HColumnDescriptor.DefaultType.TRUE, hcd.isCacheDataInL1()); + + assertEquals(HColumnDescriptor.DefaultType.NONE, hcd.isCacheDataBlockCompressed()); + hcd.setCacheDataBlockCompressed(true); + assertEquals(HColumnDescriptor.DefaultType.TRUE, hcd.isCacheDataBlockCompressed()); + + assertEquals(HColumnDescriptor.DefaultType.NONE, hcd.isEvictBlocksOnClose()); + hcd.setEvictBlocksOnClose(true); + assertEquals(HColumnDescriptor.DefaultType.TRUE, hcd.isEvictBlocksOnClose()); + + assertEquals(HColumnDescriptor.DefaultType.NONE, hcd.isPrefetchBlocksOnOpen()); + hcd.setPrefetchBlocksOnOpen(true); + assertEquals(HColumnDescriptor.DefaultType.TRUE, hcd.isPrefetchBlocksOnOpen()); + } @Test diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index d6bdec0..0334343 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -44,6 +44,13 @@ import com.google.common.annotations.VisibleForTesting; public class CacheConfig { private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName()); + + /** + * Configuration key to cache data blocks on read. + */ + public static final String CACHE_BLOCKS_ON_READ_KEY = + "hbase.rs.cacheblocksonread"; + /** * Configuration key to cache data blocks on write. There are separate * switches for bloom blocks and non-root index blocks. @@ -155,6 +162,7 @@ public class CacheConfig { } // Defaults + public static final boolean DEFAULT_CF_BLOCKCACHE_OPENED = true; public static final boolean DEFAULT_CACHE_DATA_ON_READ = true; public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false; public static final boolean DEFAULT_IN_MEMORY = false; @@ -163,10 +171,14 @@ public class CacheConfig { public static final boolean DEFAULT_EVICT_ON_CLOSE = false; public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false; public static final boolean DEFAULT_PREFETCH_ON_OPEN = false; + public static final boolean DEFAULT_CACHE_DATA_IN_L1 = false; /** Local reference to the block cache, null if completely disabled */ private final BlockCache blockCache; + /** Block cache is opend on CF Level */ + private boolean cfBlockCacheEnabled; + /** * Whether blocks should be cached on read (default is on if there is a * cache but this can be turned off on a per-family or per-request basis). @@ -214,33 +226,48 @@ public class CacheConfig { public CacheConfig(Configuration conf, HColumnDescriptor family) { this(CacheConfig.instantiateBlockCache(conf), family.isBlockCacheEnabled(), + getOption(conf.getBoolean(CACHE_BLOCKS_ON_READ_KEY, DEFAULT_CACHE_DATA_ON_READ), + family.isCacheDataOnRead()), family.isInMemory(), // For the following flags we enable them regardless of per-schema settings // if they are enabled in the global configuration. - conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, - DEFAULT_CACHE_DATA_ON_WRITE) || family.isCacheDataOnWrite(), - conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, - DEFAULT_CACHE_INDEXES_ON_WRITE) || family.isCacheIndexesOnWrite(), - conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, - DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.isCacheBloomsOnWrite(), - conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, - DEFAULT_EVICT_ON_CLOSE) || family.isEvictBlocksOnClose(), - conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED), - conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, - DEFAULT_PREFETCH_ON_OPEN) || family.isPrefetchBlocksOnOpen(), - conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1, - HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.isCacheDataInL1(), + getOption(conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, + DEFAULT_CACHE_DATA_ON_WRITE), family.isCacheDataOnWrite()), + getOption(conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, + DEFAULT_CACHE_INDEXES_ON_WRITE), family.isCacheIndexesOnWrite()), + getOption(conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, + DEFAULT_CACHE_BLOOMS_ON_WRITE), family.isCacheBloomsOnWrite()), + getOption(conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, + DEFAULT_EVICT_ON_CLOSE), family.isEvictBlocksOnClose()), + getOption(conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED), + family.isCacheDataBlockCompressed()), + getOption(conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, + DEFAULT_PREFETCH_ON_OPEN), family.isPrefetchBlocksOnOpen()), + getOption(conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1, + DEFAULT_CACHE_DATA_IN_L1), family.isCacheDataInL1()), conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT) ); } /** + * Override global setting if cf setting exist. + * */ + @VisibleForTesting + protected static boolean getOption(boolean globalSetting, HColumnDescriptor.DefaultType cfSetting) { + if (cfSetting == HColumnDescriptor.DefaultType.NONE) { + return globalSetting; + } + return Boolean.valueOf(cfSetting.toString()); + } + + /** * Create a cache configuration using the specified configuration object and * defaults for family level settings. * @param conf hbase configuration */ public CacheConfig(Configuration conf) { this(CacheConfig.instantiateBlockCache(conf), + DEFAULT_CF_BLOCKCACHE_OPENED, DEFAULT_CACHE_DATA_ON_READ, DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set // strictly from conf @@ -250,8 +277,7 @@ public class CacheConfig { conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE), conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED), conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN), - conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1, - HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1), + conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1, DEFAULT_CACHE_DATA_IN_L1), conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT) ); } @@ -272,13 +298,14 @@ public class CacheConfig { * @param cacheDataInL1 If more than one cache tier deployed, if true, cache this column families * data blocks up in the L1 tier. */ - CacheConfig(final BlockCache blockCache, + CacheConfig(final BlockCache blockCache, final boolean cfBlockCacheEnabled, final boolean cacheDataOnRead, final boolean inMemory, final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite, final boolean cacheBloomsOnWrite, final boolean evictOnClose, final boolean cacheDataCompressed, final boolean prefetchOnOpen, final boolean cacheDataInL1, final boolean dropBehindCompaction) { this.blockCache = blockCache; + this.cfBlockCacheEnabled = cfBlockCacheEnabled; this.cacheDataOnRead = cacheDataOnRead; this.inMemory = inMemory; this.cacheDataOnWrite = cacheDataOnWrite; @@ -297,10 +324,11 @@ public class CacheConfig { * @param cacheConf */ public CacheConfig(CacheConfig cacheConf) { - this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory, - cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite, - cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose, - cacheConf.cacheDataCompressed, cacheConf.prefetchOnOpen, + this(cacheConf.blockCache, cacheConf.cfBlockCacheEnabled, + cacheConf.cacheDataOnRead, cacheConf.inMemory, + cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite, + cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose, + cacheConf.cacheDataCompressed, cacheConf.prefetchOnOpen, cacheConf.cacheDataInL1, cacheConf.dropBehindCompaction); } @@ -311,6 +339,10 @@ public class CacheConfig { return this.blockCache != null; } + public boolean isCfBlockCacheEnabled() { + return isBlockCacheEnabled() && cfBlockCacheEnabled; + } + /** * Returns the block cache. * @return the block cache, or null if caching is completely disabled @@ -325,7 +357,7 @@ public class CacheConfig { * @return true if blocks should be cached on read, false if not */ public boolean shouldCacheDataOnRead() { - return isBlockCacheEnabled() && cacheDataOnRead; + return isCfBlockCacheEnabled() && cacheDataOnRead; } public boolean shouldDropBehindCompaction() { @@ -338,27 +370,31 @@ public class CacheConfig { * available. */ public boolean shouldCacheBlockOnRead(BlockCategory category) { - return isBlockCacheEnabled() - && (cacheDataOnRead || - category == BlockCategory.INDEX || + if (!isBlockCacheEnabled()) { + return false; + } + if (category == BlockCategory.INDEX || category == BlockCategory.BLOOM || (prefetchOnOpen && - (category != BlockCategory.META && - category != BlockCategory.UNKNOWN))); + (category != BlockCategory.META && + category != BlockCategory.UNKNOWN))) { + return true; + } + return shouldCacheDataOnRead(); } /** * @return true if blocks in this file should be flagged as in-memory */ public boolean isInMemory() { - return isBlockCacheEnabled() && this.inMemory; + return isCfBlockCacheEnabled() && this.inMemory; } /** * @return True if cache data blocks in L1 tier (if more than one tier in block cache deploy). */ public boolean isCacheDataInL1() { - return isBlockCacheEnabled() && this.cacheDataInL1; + return isCfBlockCacheEnabled() && this.cacheDataInL1; } /** @@ -366,7 +402,7 @@ public class CacheConfig { * written, false if not */ public boolean shouldCacheDataOnWrite() { - return isBlockCacheEnabled() && this.cacheDataOnWrite; + return isCfBlockCacheEnabled() && this.cacheDataOnWrite; } /** @@ -394,7 +430,7 @@ public class CacheConfig { * is written, false if not */ public boolean shouldCacheIndexesOnWrite() { - return isBlockCacheEnabled() && this.cacheIndexesOnWrite; + return isCfBlockCacheEnabled() && this.cacheIndexesOnWrite; } /** @@ -402,7 +438,7 @@ public class CacheConfig { * is written, false if not */ public boolean shouldCacheBloomsOnWrite() { - return isBlockCacheEnabled() && this.cacheBloomsOnWrite; + return isCfBlockCacheEnabled() && this.cacheBloomsOnWrite; } /** @@ -410,7 +446,7 @@ public class CacheConfig { * reader is closed, false if not */ public boolean shouldEvictOnClose() { - return isBlockCacheEnabled() && this.evictOnClose; + return isCfBlockCacheEnabled() && this.evictOnClose; } /** @@ -426,14 +462,14 @@ public class CacheConfig { * @return true if data blocks should be compressed in the cache, false if not */ public boolean shouldCacheDataCompressed() { - return isBlockCacheEnabled() && this.cacheDataCompressed; + return isCfBlockCacheEnabled() && this.cacheDataCompressed; } /** * @return true if this {@link BlockCategory} should be compressed in blockcache, false otherwise */ public boolean shouldCacheCompressed(BlockCategory category) { - if (!isBlockCacheEnabled()) return false; + if (!isCfBlockCacheEnabled()) return false; switch (category) { case DATA: return this.cacheDataCompressed; @@ -446,30 +482,17 @@ public class CacheConfig { * @return true if blocks should be prefetched into the cache on open, false if not */ public boolean shouldPrefetchOnOpen() { - return isBlockCacheEnabled() && this.prefetchOnOpen; + return isCfBlockCacheEnabled() && this.prefetchOnOpen; } /** * Return true if we may find this type of block in block cache. - *

- * TODO: today {@code family.isBlockCacheEnabled()} only means {@code cacheDataOnRead}, so here we - * consider lots of other configurations such as {@code cacheDataOnWrite}. We should fix this in - * the future, {@code cacheDataOnWrite} should honor the CF level {@code isBlockCacheEnabled} - * configuration. */ public boolean shouldReadBlockFromCache(BlockType blockType) { if (!isBlockCacheEnabled()) { return false; } - if (cacheDataOnRead) { - return true; - } - if (prefetchOnOpen) { - return true; - } - if (cacheDataOnWrite) { - return true; - } + //special blockType should skip CF BC checking. if (blockType == null) { return true; } @@ -477,7 +500,7 @@ public class CacheConfig { blockType.getCategory() == BlockCategory.INDEX) { return true; } - return false; + return isCfBlockCacheEnabled(); } /** @@ -497,6 +520,7 @@ public class CacheConfig { return "CacheConfig:disabled"; } return "blockCache=" + getBlockCache() + + ", cfBlockCacheEnabled=" + isCfBlockCacheEnabled() + ", cacheDataOnRead=" + shouldCacheDataOnRead() + ", cacheDataOnWrite=" + shouldCacheDataOnWrite() + ", cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java index ad08794..bee8453 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -219,6 +220,18 @@ public class TestCacheConfig { assertTrue(cc.getBlockCache() instanceof LruBlockCache); } + @Test + public void testGetOption() { + assertEquals(true, CacheConfig.getOption(true, HColumnDescriptor.DefaultType.NONE)); + assertEquals(false, CacheConfig.getOption(false, HColumnDescriptor.DefaultType.NONE)); + + assertEquals(true, CacheConfig.getOption(false, HColumnDescriptor.DefaultType.TRUE)); + assertEquals(true, CacheConfig.getOption(true, HColumnDescriptor.DefaultType.TRUE)); + + assertEquals(false, CacheConfig.getOption(true, HColumnDescriptor.DefaultType.FALSE)); + assertEquals(false, CacheConfig.getOption(false, HColumnDescriptor.DefaultType.FALSE)); + } + /** * Assert that the caches are deployed with CombinedBlockCache and of the appropriate sizes. */ diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java index 2c957ef..09090a4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java @@ -223,7 +223,7 @@ public class TestCacheOnWrite { fs = HFileSystem.get(conf); CacheConfig.GLOBAL_BLOCK_CACHE_INSTANCE = blockCache; cacheConf = - new CacheConfig(blockCache, true, true, cowType.shouldBeCached(BlockType.DATA), + new CacheConfig(blockCache, true, true, true, cowType.shouldBeCached(BlockType.DATA), cowType.shouldBeCached(BlockType.LEAF_INDEX), cowType.shouldBeCached(BlockType.BLOOM_CHUNK), false, cacheCompressedData, false, false, false); -- 1.9.3 (Apple Git-50)