From 2ca1389b27491d97071a6bc5084a1cb248fba29d Mon Sep 17 00:00:00 2001 From: Elliott Clark Date: Thu, 16 Jul 2015 02:12:08 -0700 Subject: [PATCH] HBASE-14098 Allow dropping caches behind compactions --- .../apache/hadoop/hbase/io/hfile/CacheConfig.java | 21 +++++++++++++++++---- .../org/apache/hadoop/hbase/io/hfile/HFile.java | 12 ++++++++++++ .../hadoop/hbase/io/hfile/HFileWriterImpl.java | 3 ++- .../hadoop/hbase/io/hfile/TestCacheOnWrite.java | 3 ++- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index 0ed3cbd..187755e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -131,6 +131,9 @@ public class CacheConfig { private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false; private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY="hbase.blockcache.external.class"; + private static final String DROP_BEHIND_CACHE_COMPACTION_KEY="hbase.hfile.drop.behind.compaction"; + private static final boolean DROP_BEHIND_CACHE_COMPACTION_DEFAULT = true; + /** * Enum of all built in external block caches. @@ -187,6 +190,8 @@ public class CacheConfig { /** Whether data blocks should be prefetched into the cache */ private final boolean prefetchOnOpen; + + private final boolean dropBehindCompaction; /** * If true and if more than one tier in this cache deploy -- e.g. CombinedBlockCache has an L1 * and an L2 tier -- then cache data blocks up in the L1 tier (The meta blocks are likely being @@ -218,7 +223,8 @@ public class CacheConfig { conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN) || family.isPrefetchBlocksOnOpen(), conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1, - HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.isCacheDataInL1() + HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.isCacheDataInL1(), + conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT) ); } @@ -239,7 +245,8 @@ public class CacheConfig { conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED), conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN), conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1, - HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) + HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1), + conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT) ); } @@ -264,7 +271,7 @@ public class CacheConfig { final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite, final boolean cacheBloomsOnWrite, final boolean evictOnClose, final boolean cacheDataCompressed, final boolean prefetchOnOpen, - final boolean cacheDataInL1) { + final boolean cacheDataInL1, final boolean dropBehindCompaction) { this.blockCache = blockCache; this.cacheDataOnRead = cacheDataOnRead; this.inMemory = inMemory; @@ -275,6 +282,7 @@ public class CacheConfig { this.cacheDataCompressed = cacheDataCompressed; this.prefetchOnOpen = prefetchOnOpen; this.cacheDataInL1 = cacheDataInL1; + this.dropBehindCompaction = dropBehindCompaction; LOG.info(this); } @@ -287,7 +295,7 @@ public class CacheConfig { cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite, cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose, cacheConf.cacheDataCompressed, cacheConf.prefetchOnOpen, - cacheConf.cacheDataInL1); + cacheConf.cacheDataInL1, cacheConf.dropBehindCompaction); } /** @@ -314,6 +322,11 @@ public class CacheConfig { return isBlockCacheEnabled() && cacheDataOnRead; } + + public boolean shouldDropBehindCompaction() { + return dropBehindCompaction; + } + /** * Should we cache a block of a particular category? We always cache * important blocks such as index blocks, as long as the block cache is diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index 6653c23..d3f3905 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -252,6 +252,7 @@ public class HFile { CellComparator.COMPARATOR; protected InetSocketAddress[] favoredNodes; private HFileContext fileContext; + private Boolean shouldDropBehind = true; WriterFactory(Configuration conf, CacheConfig cacheConf) { this.conf = conf; @@ -289,6 +290,11 @@ public class HFile { return this; } + public WriterFactory withDropBehind(Boolean shouldDropBehind) { + this.shouldDropBehind = shouldDropBehind; + return this; + } + public Writer create() throws IOException { if ((path != null ? 1 : 0) + (ostream != null ? 1 : 0) != 1) { throw new AssertionError("Please specify exactly one of " + @@ -296,7 +302,13 @@ public class HFile { } if (path != null) { ostream = HFileWriterImpl.createOutputStream(conf, fs, path, favoredNodes); + try { + ostream.setDropBehind(cacheConf.shouldDropBehindCompaction()); + } catch (UnsupportedOperationException uoe) { + LOG.debug("Unable to set drop behind on " + path, uoe); + } } + return createWriter(fs, path, ostream, comparator, fileContext); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java index 6a5a9d1..d199928 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java @@ -268,7 +268,8 @@ public class HFileWriterImpl implements HFile.Writer { FileSystem fs, Path path, InetSocketAddress[] favoredNodes) throws IOException { FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY); - return FSUtils.create(fs, path, perms, favoredNodes); + FSDataOutputStream fdos = FSUtils.create(fs, path, perms, favoredNodes); + return fdos; } /** Additional initialization steps */ diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java index 77311bc..cdc860f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java @@ -253,7 +253,8 @@ public class TestCacheOnWrite { cacheConf = new CacheConfig(blockCache, true, true, cowType.shouldBeCached(BlockType.DATA), cowType.shouldBeCached(BlockType.LEAF_INDEX), - cowType.shouldBeCached(BlockType.BLOOM_CHUNK), false, cacheCompressedData, false, false); + cowType.shouldBeCached(BlockType.BLOOM_CHUNK), false, cacheCompressedData, + false, false, false); } @After -- 2.4.3