Index: src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java (working copy) @@ -31,9 +31,10 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.Compression; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileScanner; -import org.apache.hadoop.hbase.io.hfile.Compression; import org.apache.hadoop.hbase.util.Bytes; /** @@ -188,9 +189,9 @@ @Override void setUp() throws Exception { - writer = HFile.getWriterFactory(conf).createWriter(this.fs, this.mf, - RFILE_BLOCKSIZE, - (Compression.Algorithm) null, null); + writer = + HFile.getWriterFactory(conf).createWriter(this.fs, + this.mf, RFILE_BLOCKSIZE, (Compression.Algorithm) null, null); } @Override @@ -226,7 +227,7 @@ @Override void setUp() throws Exception { - reader = HFile.createReader(this.fs, this.mf, null, false, false); + reader = HFile.createReader(this.fs, this.mf, new CacheConfig(this.conf)); this.reader.loadFileInfo(); } Index: src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java (working copy) @@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -31,18 +32,18 @@ import org.apache.hadoop.hbase.HBaseTestCase; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.hfile.BlockCache; -import org.apache.hadoop.hbase.io.hfile.CacheStats; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.LruBlockCache; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; - import org.junit.Test; public class TestBlocksRead extends HBaseTestCase { @@ -98,7 +99,7 @@ HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); Path path = new Path(DIR + callingMethod); region = HRegion.createHRegion(info, path, conf, htd); - blockCache = StoreFile.getBlockCache(conf); + blockCache = new CacheConfig(conf).getBlockCache(); } private void putData(byte[] cf, String row, String col, long version) @@ -178,6 +179,10 @@ return blockCache.getStats().getRequestCount(); } + private static long getBlkCount() { + return blockCache.getBlockCount(); + } + /** * Test # of blocks read for some simple seek cases. * @throws Exception @@ -316,4 +321,49 @@ verifyData(kvs[1], "row", "col2", 12); verifyData(kvs[2], "row", "col3", 13); } + + /** + * Test # of blocks read to ensure disabling cache-fill on Scan works. + * @throws Exception + */ + @Test + public void testBlocksStoredWhenCachingDisabled() throws Exception { + byte [] TABLE = Bytes.toBytes("testBlocksReadWhenCachingDisabled"); + byte [] FAMILY = Bytes.toBytes("cf1"); + byte [][] FAMILIES = new byte[][] { FAMILY }; + + HBaseConfiguration conf = getConf(); + initHRegion(TABLE, getName(), conf, FAMILIES); + + putData(FAMILY, "row", "col1", 1); + putData(FAMILY, "row", "col2", 2); + region.flushcache(); + + // Execute a scan with caching turned off + // Expected blocks stored: 0 + long blocksStart = getBlkCount(); + Scan scan = new Scan(); + scan.setCacheBlocks(false); + RegionScanner rs = region.getScanner(scan); + List result = new ArrayList(2); + rs.next(result); + assertEquals(2, result.size()); + rs.close(); + long blocksEnd = getBlkCount(); + + assertEquals(blocksStart, blocksEnd); + + // Execute with caching turned on + // Expected blocks stored: 2 + blocksStart = blocksEnd; + scan.setCacheBlocks(true); + rs = region.getScanner(scan); + result = new ArrayList(2); + rs.next(result); + assertEquals(2, result.size()); + rs.close(); + blocksEnd = getBlkCount(); + + assertEquals(2, blocksEnd - blocksStart); + } } Index: src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java (working copy) @@ -36,9 +36,8 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.regionserver.StoreFile.Reader; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.regionserver.wal.HLog; -import org.apache.hadoop.hbase.regionserver.wal.TestWALReplay; import org.apache.hadoop.hbase.util.Bytes; import com.google.common.collect.Lists; @@ -100,8 +99,9 @@ boolean isRef = false; MockStoreFile(long length, boolean isRef) throws IOException { - super(TEST_UTIL.getTestFileSystem(), TEST_FILE, false, - TEST_UTIL.getConfiguration(), BloomType.NONE, false); + super(TEST_UTIL.getTestFileSystem(), TEST_FILE, + TEST_UTIL.getConfiguration(), + new CacheConfig(TEST_UTIL.getConfiguration()), BloomType.NONE); this.length = length; this.isRef = isRef; } Index: src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java (working copy) @@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; import org.apache.hadoop.hbase.util.Bytes; @@ -70,13 +71,14 @@ HBaseTestingUtility.getTestDir("internalScannerExposesErrors"), "regionname"), "familyname"); FaultyFileSystem fs = new FaultyFileSystem(util.getTestFileSystem()); + CacheConfig cacheConf = new CacheConfig(util.getConfiguration()); StoreFile.Writer writer = StoreFile.createWriter(fs, hfilePath, 2*1024, - util.getConfiguration()); + util.getConfiguration(), cacheConf); TestStoreFile.writeStoreFile( writer, Bytes.toBytes("cf"), Bytes.toBytes("qual")); - StoreFile sf = new StoreFile(fs, writer.getPath(), false, - util.getConfiguration(), StoreFile.BloomType.NONE, false); + StoreFile sf = new StoreFile(fs, writer.getPath(), + util.getConfiguration(), cacheConf, StoreFile.BloomType.NONE); StoreFile.Reader reader = sf.createReader(); HFileScanner scanner = reader.getScanner(false, true); @@ -112,13 +114,14 @@ HBaseTestingUtility.getTestDir("internalScannerExposesErrors"), "regionname"), "familyname"); FaultyFileSystem fs = new FaultyFileSystem(util.getTestFileSystem()); + CacheConfig cacheConf = new CacheConfig(util.getConfiguration()); StoreFile.Writer writer = StoreFile.createWriter(fs, hfilePath, 2 * 1024, - util.getConfiguration()); + util.getConfiguration(), cacheConf); TestStoreFile.writeStoreFile( writer, Bytes.toBytes("cf"), Bytes.toBytes("qual")); - StoreFile sf = new StoreFile(fs, writer.getPath(), false, - util.getConfiguration(), BloomType.NONE, false); + StoreFile sf = new StoreFile(fs, writer.getPath(), util.getConfiguration(), + cacheConf, BloomType.NONE); List scanners = StoreFileScanner.getScannersForStoreFiles( Collections.singletonList(sf), false, true); KeyValueScanner scanner = scanners.get(0); Index: src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java (working copy) @@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.regionserver.FlushRequester; @@ -199,7 +200,8 @@ HLog wal = createWAL(this.conf); HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf); Path f = new Path(basedir, "hfile"); - HFile.Writer writer = HFile.getWriterFactory(conf).createWriter(this.fs, f); + HFile.Writer writer = + HFile.getWriterFactory(conf).createWriter(this.fs, f); byte [] family = htd.getFamilies().iterator().next().getName(); byte [] row = Bytes.toBytes(tableNameStr); writer.append(new KeyValue(row, family, family, row)); Index: src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileBlockCacheSummary.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileBlockCacheSummary.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileBlockCacheSummary.java (working copy) @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.hfile.BlockCache; import org.apache.hadoop.hbase.io.hfile.BlockCacheColumnFamilySummary; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.util.Bytes; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -100,7 +101,8 @@ scan(ht, FAMILY); scan(ht2, FAMILY); - BlockCache bc = StoreFile.getBlockCache(TEST_UTIL.getConfiguration()); + BlockCache bc = + new CacheConfig(TEST_UTIL.getConfiguration()).getBlockCache(); List bcs = bc.getBlockCacheColumnFamilySummaries(TEST_UTIL.getConfiguration()); LOG.info("blockCacheSummary: " + bcs); Index: src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java (working copy) @@ -20,7 +20,10 @@ package org.apache.hadoop.hbase.regionserver; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.IOException; import java.util.ArrayList; @@ -38,6 +41,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.TestHFileWriterV2; import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; @@ -105,6 +109,7 @@ } private static Configuration conf; + private static CacheConfig cacheConf; private FileSystem fs; private BlockCache blockCache; @@ -123,7 +128,8 @@ fs = FileSystem.get(conf); - blockCache = StoreFile.getBlockCache(conf); + cacheConf = new CacheConfig(conf); + blockCache = cacheConf.getBlockCache(); assertNotNull(blockCache); } @@ -187,7 +193,7 @@ private void readStoreFile(int t, BloomType bt, List kvs, Path sfPath) throws IOException { - StoreFile sf = new StoreFile(fs, sfPath, true, conf, bt, false); + StoreFile sf = new StoreFile(fs, sfPath, conf, cacheConf, bt); StoreFile.Reader r = sf.createReader(); final boolean pread = true; // does not really matter StoreFileScanner scanner = r.getStoreFileScanner(true, pread); @@ -283,11 +289,12 @@ throws IOException { conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_BLOCK_SIZE, BLOOM_BLOCK_SIZES[t]); - conf.setBoolean(HFile.CACHE_BLOCKS_ON_WRITE_KEY, true); + conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true); + cacheConf = new CacheConfig(conf); StoreFile.Writer w = StoreFile.createWriter(fs, HBaseTestingUtility.getTestDir(), BLOCK_SIZES[t], null, null, conf, - bt, 0); + cacheConf, bt, 0); assertTrue(w.hasBloom()); assertTrue(w.getBloomWriter() instanceof CompoundBloomFilterWriter); Index: src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java (working copy) @@ -51,6 +51,7 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.regionserver.wal.HLog; import org.apache.hadoop.hbase.security.User; @@ -209,7 +210,7 @@ Configuration c = HBaseConfiguration.create(); FileSystem fs = FileSystem.get(c); StoreFile.Writer w = StoreFile.createWriter(fs, storedir, - StoreFile.DEFAULT_BLOCKSIZE_SMALL, c); + StoreFile.DEFAULT_BLOCKSIZE_SMALL, c, new CacheConfig(c)); w.appendMetadata(seqid + 1, false); w.close(); this.store.close(); Index: src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java (working copy) @@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.Reference.Range; import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.CacheStats; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileScanner; @@ -58,6 +59,7 @@ public class TestStoreFile extends HBaseTestCase { static final Log LOG = LogFactory.getLog(TestStoreFile.class); private MiniDFSCluster cluster; + private CacheConfig cacheConf; @Override public void setUp() throws Exception { @@ -66,6 +68,7 @@ // Set the hbase.rootdir to be the home directory in mini dfs. this.conf.set(HConstants.HBASE_DIR, this.cluster.getFileSystem().getHomeDirectory().toString()); + this.cacheConf = new CacheConfig(conf); } catch (IOException e) { shutdownDfs(cluster); } @@ -89,10 +92,10 @@ // Make up a directory hierarchy that has a regiondir and familyname. StoreFile.Writer writer = StoreFile.createWriter(this.fs, new Path(new Path(this.testDir, "regionname"), "familyname"), 2 * 1024, - conf); + conf, cacheConf); writeStoreFile(writer); - checkHalfHFile(new StoreFile(this.fs, writer.getPath(), true, conf, - StoreFile.BloomType.NONE, false)); + checkHalfHFile(new StoreFile(this.fs, writer.getPath(), conf, cacheConf, + StoreFile.BloomType.NONE)); } private void writeStoreFile(final StoreFile.Writer writer) throws IOException { @@ -130,10 +133,10 @@ Path dir = new Path(storedir, "1234567890"); // Make a store file and write data to it. StoreFile.Writer writer = StoreFile.createWriter(this.fs, dir, 8 * 1024, - conf); + conf, cacheConf); writeStoreFile(writer); - StoreFile hsf = new StoreFile(this.fs, writer.getPath(), true, conf, - StoreFile.BloomType.NONE, false); + StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf, + StoreFile.BloomType.NONE); StoreFile.Reader reader = hsf.createReader(); // Split on a row, not in middle of row. Midkey returned by reader // may be in middle of row. Create new one with empty column and @@ -144,8 +147,8 @@ byte [] finalRow = kv.getRow(); // Make a reference Path refPath = StoreFile.split(fs, dir, hsf, midRow, Range.top); - StoreFile refHsf = new StoreFile(this.fs, refPath, true, conf, - StoreFile.BloomType.NONE, false); + StoreFile refHsf = new StoreFile(this.fs, refPath, conf, cacheConf, + StoreFile.BloomType.NONE); // Now confirm that I can read from the reference and that it only gets // keys from top half of the file. HFileScanner s = refHsf.createReader().getScanner(false, false); @@ -181,10 +184,10 @@ Path bottomPath = StoreFile.split(this.fs, bottomDir, f, midRow, Range.bottom); // Make readers on top and bottom. - StoreFile.Reader top = new StoreFile(this.fs, topPath, true, conf, - StoreFile.BloomType.NONE, false).createReader(); - StoreFile.Reader bottom = new StoreFile(this.fs, bottomPath, true, conf, - StoreFile.BloomType.NONE, false).createReader(); + StoreFile.Reader top = new StoreFile(this.fs, topPath, conf, cacheConf, + StoreFile.BloomType.NONE).createReader(); + StoreFile.Reader bottom = new StoreFile(this.fs, bottomPath, conf, cacheConf, + StoreFile.BloomType.NONE).createReader(); ByteBuffer previous = null; LOG.info("Midkey: " + midKV.toString()); ByteBuffer bbMidkeyBytes = ByteBuffer.wrap(midkey); @@ -239,10 +242,10 @@ topPath = StoreFile.split(this.fs, topDir, f, badmidkey, Range.top); bottomPath = StoreFile.split(this.fs, bottomDir, f, badmidkey, Range.bottom); - top = new StoreFile(this.fs, topPath, true, conf, - StoreFile.BloomType.NONE, false).createReader(); - bottom = new StoreFile(this.fs, bottomPath, true, conf, - StoreFile.BloomType.NONE, false).createReader(); + top = new StoreFile(this.fs, topPath, conf, cacheConf, + StoreFile.BloomType.NONE).createReader(); + bottom = new StoreFile(this.fs, bottomPath, conf, cacheConf, + StoreFile.BloomType.NONE).createReader(); bottomScanner = bottom.getScanner(false, false); int count = 0; while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) || @@ -284,10 +287,10 @@ topPath = StoreFile.split(this.fs, topDir, f, badmidkey, Range.top); bottomPath = StoreFile.split(this.fs, bottomDir, f, badmidkey, Range.bottom); - top = new StoreFile(this.fs, topPath, true, conf, - StoreFile.BloomType.NONE, false).createReader(); - bottom = new StoreFile(this.fs, bottomPath, true, conf, - StoreFile.BloomType.NONE, false).createReader(); + top = new StoreFile(this.fs, topPath, conf, cacheConf, + StoreFile.BloomType.NONE).createReader(); + bottom = new StoreFile(this.fs, bottomPath, conf, cacheConf, + StoreFile.BloomType.NONE).createReader(); first = true; bottomScanner = bottom.getScanner(false, false); while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) || @@ -345,7 +348,7 @@ } writer.close(); - StoreFile.Reader reader = new StoreFile.Reader(fs, f, null, false, false); + StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf); reader.loadFileInfo(); reader.loadBloomfilter(); StoreFileScanner scanner = reader.getStoreFileScanner(false, false); @@ -386,7 +389,7 @@ Path f = new Path(ROOT_DIR, getName()); StoreFile.Writer writer = new StoreFile.Writer(fs, f, StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM, - conf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW, 2000); + conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW, 2000); bloomWriteRead(writer, fs); } @@ -413,11 +416,11 @@ for (int x : new int[]{0,1}) { // write the file - Path f = new Path(ROOT_DIR, getName()); + Path f = new Path(ROOT_DIR, getName() + x); StoreFile.Writer writer = new StoreFile.Writer(fs, f, StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM, - conf, KeyValue.COMPARATOR, bt[x], expKeys[x]); + conf, cacheConf, KeyValue.COMPARATOR, bt[x], expKeys[x]); long now = System.currentTimeMillis(); for (int i = 0; i < rowCount*2; i += 2) { // rows @@ -434,10 +437,10 @@ } writer.close(); - StoreFile.Reader reader = new StoreFile.Reader(fs, f, null, false, false); + StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf); reader.loadFileInfo(); reader.loadBloomfilter(); - StoreFileScanner scanner = reader.getStoreFileScanner(false, false); + StoreFileScanner scanner = reader.getStoreFileScanner(true, true); assertEquals(expKeys[x], reader.bloomFilter.getKeyCount()); // check false positives rate @@ -487,7 +490,7 @@ // this should not create a bloom because the max keys is too small StoreFile.Writer writer = new StoreFile.Writer(fs, f, StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM, - conf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW, 2000); + conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW, 2000); assertFalse(writer.hasBloom()); writer.close(); fs.delete(f, true); @@ -510,7 +513,8 @@ // because Java can't create a contiguous array > MAX_INT writer = new StoreFile.Writer(fs, f, StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM, - conf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW, Integer.MAX_VALUE); + conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW, + Integer.MAX_VALUE); assertFalse(writer.hasBloom()); writer.close(); fs.delete(f, true); @@ -602,7 +606,7 @@ "familyname"); Path dir = new Path(storedir, "1234567890"); StoreFile.Writer writer = StoreFile.createWriter(this.fs, dir, 8 * 1024, - conf); + conf, cacheConf); List kvList = getKeyValueSet(timestamps,numRows, family, qualifier); @@ -613,8 +617,8 @@ writer.appendMetadata(0, false); writer.close(); - StoreFile hsf = new StoreFile(this.fs, writer.getPath(), true, conf, - StoreFile.BloomType.NONE, false); + StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf, + StoreFile.BloomType.NONE); StoreFile.Reader reader = hsf.createReader(); StoreFileScanner scanner = reader.getStoreFileScanner(false, false); TreeSet columns = new TreeSet(); @@ -640,14 +644,13 @@ public void testCacheOnWriteEvictOnClose() throws Exception { Configuration conf = this.conf; - conf.setBoolean("hbase.rs.evictblocksonclose", false); // Find a home for our files Path baseDir = new Path(new Path(this.testDir, "regionname"), "twoCOWEOC"); // Grab the block cache and get the initial hit/miss counts - BlockCache bc = StoreFile.getBlockCache(conf); + BlockCache bc = new CacheConfig(conf).getBlockCache(); assertNotNull(bc); CacheStats cs = bc.getStats(); long startHit = cs.getHitCount(); @@ -655,11 +658,12 @@ long startEvicted = cs.getEvictedCount(); // Let's write a StoreFile with three blocks, with cache on write off - conf.setBoolean(HFile.CACHE_BLOCKS_ON_WRITE_KEY, false); + conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false); + CacheConfig cacheConf = new CacheConfig(conf); Path pathCowOff = new Path(baseDir, "123456789"); - StoreFile.Writer writer = writeStoreFile(conf, pathCowOff, 3); - StoreFile hsf = new StoreFile(this.fs, writer.getPath(), true, conf, - StoreFile.BloomType.NONE, false); + StoreFile.Writer writer = writeStoreFile(conf, cacheConf, pathCowOff, 3); + StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf, + StoreFile.BloomType.NONE); LOG.debug(hsf.getPath().toString()); // Read this file, we should see 3 misses @@ -676,11 +680,12 @@ reader.close(); // Now write a StoreFile with three blocks, with cache on write on - conf.setBoolean(HFile.CACHE_BLOCKS_ON_WRITE_KEY, true); + conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true); + cacheConf = new CacheConfig(conf); Path pathCowOn = new Path(baseDir, "123456788"); - writer = writeStoreFile(conf, pathCowOn, 3); - hsf = new StoreFile(this.fs, writer.getPath(), true, conf, - StoreFile.BloomType.NONE, false); + writer = writeStoreFile(conf, cacheConf, pathCowOn, 3); + hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf, + StoreFile.BloomType.NONE); // Read this file, we should see 3 hits reader = hsf.createReader(); @@ -695,14 +700,14 @@ reader.close(); // Let's read back the two files to ensure the blocks exactly match - hsf = new StoreFile(this.fs, pathCowOff, true, conf, - StoreFile.BloomType.NONE, false); + hsf = new StoreFile(this.fs, pathCowOff, conf, cacheConf, + StoreFile.BloomType.NONE); StoreFile.Reader readerOne = hsf.createReader(); readerOne.loadFileInfo(); StoreFileScanner scannerOne = readerOne.getStoreFileScanner(true, true); scannerOne.seek(KeyValue.LOWESTKEY); - hsf = new StoreFile(this.fs, pathCowOn, true, conf, - StoreFile.BloomType.NONE, false); + hsf = new StoreFile(this.fs, pathCowOn, conf, cacheConf, + StoreFile.BloomType.NONE); StoreFile.Reader readerTwo = hsf.createReader(); readerTwo.loadFileInfo(); StoreFileScanner scannerTwo = readerTwo.getStoreFileScanner(true, true); @@ -731,8 +736,9 @@ // Let's close the first file with evict on close turned on conf.setBoolean("hbase.rs.evictblocksonclose", true); - hsf = new StoreFile(this.fs, pathCowOff, true, conf, - StoreFile.BloomType.NONE, false); + cacheConf = new CacheConfig(conf); + hsf = new StoreFile(this.fs, pathCowOff, conf, cacheConf, + StoreFile.BloomType.NONE); reader = hsf.createReader(); reader.close(); @@ -744,8 +750,9 @@ // Let's close the second file with evict on close turned off conf.setBoolean("hbase.rs.evictblocksonclose", false); - hsf = new StoreFile(this.fs, pathCowOn, true, conf, - StoreFile.BloomType.NONE, false); + cacheConf = new CacheConfig(conf); + hsf = new StoreFile(this.fs, pathCowOn, conf, cacheConf, + StoreFile.BloomType.NONE); reader = hsf.createReader(); reader.close(); @@ -755,8 +762,8 @@ assertEquals(startEvicted, cs.getEvictedCount()); } - private StoreFile.Writer writeStoreFile(Configuration conf, Path path, - int numBlocks) + private StoreFile.Writer writeStoreFile(Configuration conf, + CacheConfig cacheConf, Path path, int numBlocks) throws IOException { // Let's put ~5 small KVs in each block, so let's make 5*numBlocks KVs int numKVs = 5 * numBlocks; @@ -771,7 +778,7 @@ int blockSize = totalSize / numBlocks; StoreFile.Writer writer = new StoreFile.Writer(fs, path, blockSize, HFile.DEFAULT_COMPRESSION_ALGORITHM, - conf, KeyValue.COMPARATOR, StoreFile.BloomType.NONE, 2000); + conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.NONE, 2000); // We'll write N-1 KVs to ensure we don't write an extra block kvs.remove(kvs.size()-1); for (KeyValue kv : kvs) { Index: src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java (working copy) @@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.Compression; import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm; import org.apache.hadoop.hbase.io.hfile.HFile; @@ -281,8 +282,8 @@ FileStatus[] file = fs.listStatus(sub3[0].getPath()); // open as HFile Reader and pull out TIMERANGE FileInfo. - HFile.Reader rd = HFile.createReader(fs, file[0].getPath(), null, true, - false); + HFile.Reader rd = HFile.createReader(fs, file[0].getPath(), + new CacheConfig(conf)); Map finfo = rd.loadFileInfo(); byte[] range = finfo.get("TIMERANGE".getBytes()); assertNotNull(range); @@ -608,8 +609,8 @@ // verify that the compression on this file matches the configured // compression Path dataFilePath = fileSystem.listStatus(f.getPath())[0].getPath(); - Reader reader = HFile.createReader(fileSystem, dataFilePath, null, - false, true); + Reader reader = HFile.createReader(fileSystem, dataFilePath, + new CacheConfig(conf)); reader.loadFileInfo(); assertEquals("Incorrect compression used for column family " + familyStr + "(reader: " + reader + ")", Index: src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java (working copy) @@ -19,10 +19,11 @@ */ package org.apache.hadoop.hbase.mapreduce; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.io.IOException; -import java.util.Arrays; import java.util.TreeMap; import org.apache.hadoop.conf.Configuration; @@ -34,6 +35,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.Compression; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileScanner; @@ -41,8 +43,6 @@ import org.apache.hadoop.hbase.util.Bytes; import org.junit.Test; -import static org.junit.Assert.*; - /** * Test cases for the "load" half of the HFileOutputFormat bulk load * functionality. These tests run faster than the full MR cluster @@ -179,7 +179,7 @@ private int verifyHFile(Path p) throws IOException { Configuration conf = util.getConfiguration(); HFile.Reader reader = HFile.createReader( - p.getFileSystem(conf), p, null, false, false); + p.getFileSystem(conf), p, new CacheConfig(conf)); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, false); scanner.seekTo(); @@ -203,7 +203,8 @@ byte[] family, byte[] qualifier, byte[] startKey, byte[] endKey, int numRows) throws IOException { - HFile.Writer writer = HFile.getWriterFactory(conf).createWriter(fs, path, + HFile.Writer writer = + HFile.getWriterFactory(conf, new CacheConfig(conf)).createWriter(fs, path, BLOCKSIZE, COMPRESSION, KeyValue.KEY_COMPARATOR); long now = System.currentTimeMillis(); Index: src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java (working copy) @@ -118,7 +118,8 @@ long totalBytes = 0; FSDataOutputStream fout = createFSOutput(path, fs); try { - Writer writer = HFile.getWriterFactory(conf).createWriter(fout, + Writer writer = + HFile.getWriterFactory(conf).createWriter(fout, options.minBlockSize, options.compress, null); try { BytesWritable key = new BytesWritable(); @@ -164,7 +165,7 @@ long totalBytes = 0; FSDataInputStream fsdis = fs.open(path); Reader reader = HFile.createReader(path, fsdis, - fs.getFileStatus(path).getLen(), null, false, false); + fs.getFileStatus(path).getLen(), new CacheConfig(conf)); reader.loadFileInfo(); KeySampler kSampler = new KeySampler(rng, reader.getFirstKey(), reader.getLastKey(), Index: src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java (working copy) @@ -157,7 +157,8 @@ if ("HFile".equals(fileType)){ System.out.println("HFile write method: "); - HFile.Writer writer = HFile.getWriterFactory(conf).createWriter(fout, + HFile.Writer writer = + HFile.getWriterFactory(conf).createWriter(fout, minBlockSize, codecName, null); // Writing value in one shot. @@ -237,7 +238,7 @@ if ("HFile".equals(fileType)){ HFile.Reader reader = HFile.createReader(path, fs.open(path), - fs.getFileStatus(path).getLen(), null, false, false); + fs.getFileStatus(path).getLen(), new CacheConfig(conf)); reader.loadFileInfo(); switch (method) { Index: src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java (working copy) @@ -64,6 +64,7 @@ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private Configuration conf; + private CacheConfig cacheConf; private FileSystem fs; private Random rand = new Random(12983177L); private Path storeFilePath; @@ -82,11 +83,11 @@ KeyValue.Type.values().length - 2; private static enum CacheOnWriteType { - DATA_BLOCKS(BlockType.DATA, HFile.CACHE_BLOCKS_ON_WRITE_KEY), + DATA_BLOCKS(BlockType.DATA, CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY), BLOOM_BLOCKS(BlockType.BLOOM_CHUNK, - BloomFilterFactory.IO_STOREFILE_BLOOM_CACHE_ON_WRITE), + CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY), INDEX_BLOCKS(BlockType.LEAF_INDEX, - HFileBlockIndex.CACHE_INDEX_BLOCKS_ON_WRITE_KEY); + CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY); private final String confKey; private final BlockType inlineBlockType; @@ -114,6 +115,7 @@ this.cowType = cowType; this.compress = compress; testName = "[cacheOnWrite=" + cowType + ", compress=" + compress + "]"; + System.out.println(testName); } @Parameters @@ -134,9 +136,17 @@ conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, INDEX_BLOCK_SIZE); conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_BLOCK_SIZE, BLOOM_BLOCK_SIZE); + conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, + cowType.shouldBeCached(BlockType.DATA)); + conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, + cowType.shouldBeCached(BlockType.LEAF_INDEX)); + conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, + cowType.shouldBeCached(BlockType.BLOOM_CHUNK)); cowType.modifyConf(conf); fs = FileSystem.get(conf); - blockCache = StoreFile.getBlockCache(conf); + cacheConf = new CacheConfig(conf); + blockCache = cacheConf.getBlockCache(); + System.out.println("setUp()"); } @After @@ -152,7 +162,7 @@ private void readStoreFile() throws IOException { HFileReaderV2 reader = (HFileReaderV2) HFile.createReader(fs, - storeFilePath, null, false, false); + storeFilePath, cacheConf); LOG.info("HFile information: " + reader); HFileScanner scanner = reader.getScanner(false, false); assertTrue(testName, scanner.seekTo()); @@ -167,8 +177,8 @@ if (prevBlock != null) { onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader(); } - // Flags: cache the block, use pread, this is not a compaction. - HFileBlock block = reader.readBlock(offset, onDiskSize, true, true, + // Flags: don't cache the block, use pread, this is not a compaction. + HFileBlock block = reader.readBlock(offset, onDiskSize, false, true, false); String blockCacheKey = HFile.getBlockCacheKey(reader.getName(), offset); boolean isCached = blockCache.getBlock(blockCacheKey, true) != null; @@ -210,7 +220,7 @@ "test_cache_on_write"); StoreFile.Writer sfw = StoreFile.createWriter(fs, storeFileParentDir, DATA_BLOCK_SIZE, compress, KeyValue.COMPARATOR, conf, - StoreFile.BloomType.ROWCOL, NUM_KV); + cacheConf, StoreFile.BloomType.ROWCOL, NUM_KV); final int rowLen = 32; for (int i = 0; i < NUM_KV; ++i) { Index: src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java (working copy) @@ -56,7 +56,7 @@ "8e8ab58dcf39412da19833fcd8f687ac"); Path existingHFilePath = new Path(url.getPath()); HFile.Reader reader = - HFile.createReader(fs, existingHFilePath, null, false, false); + HFile.createReader(fs, existingHFilePath, new CacheConfig(conf)); reader.loadFileInfo(); FixedFileTrailer trailer = reader.getTrailer(); Index: src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java (working copy) @@ -42,8 +42,10 @@ Path ncTFile = new Path(HBaseTestingUtility.getTestDir(), "basic.hfile"); FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile); + CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration()); HFile.Writer writer = HFile.getWriterFactory( - TEST_UTIL.getConfiguration()).createWriter(fout, 4000, "none", null); + TEST_UTIL.getConfiguration(), cacheConf).createWriter( + fout, 4000, "none", null); int numberOfKeys = 1000; String valueString = "Value"; @@ -61,7 +63,7 @@ fout.close(); HFile.Reader reader = HFile.createReader(TEST_UTIL.getTestFileSystem(), - ncTFile, null, false, false); + ncTFile, cacheConf); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, true); Index: src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java (working copy) @@ -53,6 +53,7 @@ HBaseTestingUtility.getTestDir("TestHFile").toString(); private final int minBlockSize = 512; private static String localFormatter = "%010d"; + private static CacheConfig cacheConf = null; /** * Test empty HFile. @@ -60,10 +61,11 @@ * @throws IOException */ public void testEmptyHFile() throws IOException { + if (cacheConf == null) cacheConf = new CacheConfig(conf); Path f = new Path(ROOT_DIR, getName()); - Writer w = HFile.getWriterFactory(conf).createWriter(this.fs, f); + Writer w = HFile.getWriterFactory(conf, cacheConf).createWriter(this.fs, f); w.close(); - Reader r = HFile.createReader(fs, f, null, false, false); + Reader r = HFile.createReader(fs, f, cacheConf); r.loadFileInfo(); assertNull(r.getFirstKey()); assertNull(r.getLastKey()); @@ -130,16 +132,18 @@ * test none codecs */ void basicWithSomeCodec(String codec) throws IOException { - Path ncTFile = new Path(ROOT_DIR, "basic.hfile"); + if (cacheConf == null) cacheConf = new CacheConfig(conf); + Path ncTFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString()); FSDataOutputStream fout = createFSOutput(ncTFile); - Writer writer = HFile.getWriterFactory(conf).createWriter(fout, + Writer writer = HFile.getWriterFactory(conf, cacheConf).createWriter(fout, minBlockSize, Compression.getCompressionAlgorithmByName(codec), null); LOG.info(writer); writeRecords(writer); fout.close(); FSDataInputStream fin = fs.open(ncTFile); Reader reader = HFile.createReader(ncTFile, fs.open(ncTFile), - fs.getFileStatus(ncTFile).getLen(), null, false, false); + fs.getFileStatus(ncTFile).getLen(), cacheConf); + System.out.println(cacheConf.toString()); // Load up the index. reader.loadFileInfo(); // Get a scanner that caches and that does not use pread. @@ -205,9 +209,10 @@ } private void metablocks(final String compress) throws Exception { + if (cacheConf == null) cacheConf = new CacheConfig(conf); Path mFile = new Path(ROOT_DIR, "meta.hfile"); FSDataOutputStream fout = createFSOutput(mFile); - Writer writer = HFile.getWriterFactory(conf).createWriter(fout, + Writer writer = HFile.getWriterFactory(conf, cacheConf).createWriter(fout, minBlockSize, Compression.getCompressionAlgorithmByName(compress), null); someTestingWithMetaBlock(writer); @@ -215,7 +220,7 @@ fout.close(); FSDataInputStream fin = fs.open(mFile); Reader reader = HFile.createReader(mFile, fs.open(mFile), - this.fs.getFileStatus(mFile).getLen(), null, false, false); + this.fs.getFileStatus(mFile).getLen(), cacheConf); reader.loadFileInfo(); // No data -- this should return false. assertFalse(reader.getScanner(false, false).seekTo()); @@ -232,16 +237,17 @@ } public void testNullMetaBlocks() throws Exception { + if (cacheConf == null) cacheConf = new CacheConfig(conf); for (Compression.Algorithm compressAlgo : HBaseTestingUtility.COMPRESSION_ALGORITHMS) { Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile"); FSDataOutputStream fout = createFSOutput(mFile); - Writer writer = HFile.getWriterFactory(conf).createWriter(fout, + Writer writer = HFile.getWriterFactory(conf, cacheConf).createWriter(fout, minBlockSize, compressAlgo, null); writer.append("foo".getBytes(), "value".getBytes()); writer.close(); fout.close(); - Reader reader = HFile.createReader(fs, mFile, null, false, false); + Reader reader = HFile.createReader(fs, mFile, cacheConf); reader.loadFileInfo(); assertNull(reader.getMetaBlock("non-existant", false)); } @@ -257,9 +263,10 @@ } public void testComparator() throws IOException { + if (cacheConf == null) cacheConf = new CacheConfig(conf); Path mFile = new Path(ROOT_DIR, "meta.tfile"); FSDataOutputStream fout = createFSOutput(mFile); - Writer writer = HFile.getWriterFactory(conf).createWriter(fout, + Writer writer = HFile.getWriterFactory(conf, cacheConf).createWriter(fout, minBlockSize, (Compression.Algorithm) null, new KeyComparator() { @Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, Index: src/test/java/org/apache/hadoop/hbase/io/hfile/RandomSeek.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/io/hfile/RandomSeek.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/io/hfile/RandomSeek.java (working copy) @@ -67,7 +67,10 @@ Path path = new Path("/Users/ryan/rfile.big.txt"); long start = System.currentTimeMillis(); SimpleBlockCache cache = new SimpleBlockCache(); - Reader reader = HFile.createReader(lfs, path, cache, false, false); + CacheConfig cacheConf = new CacheConfig(cache, true, false, false, false, + false, false, false); + + Reader reader = HFile.createReader(lfs, path, cacheConf); reader.loadFileInfo(); System.out.println(reader.getTrailer()); long end = System.currentTimeMillis(); Index: src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java (working copy) @@ -460,7 +460,8 @@ public void testHFileWriterAndReader() throws IOException { Path hfilePath = new Path(HBaseTestingUtility.getTestDir(), "hfile_for_block_index"); - BlockCache blockCache = StoreFile.getBlockCache(conf); + CacheConfig cacheConf = new CacheConfig(conf); + BlockCache blockCache = cacheConf.getBlockCache(); for (int testI = 0; testI < INDEX_CHUNK_SIZES.length; ++testI) { int indexBlockSize = INDEX_CHUNK_SIZES[testI]; @@ -478,7 +479,8 @@ // Write the HFile { - HFile.Writer writer = HFile.getWriterFactory(conf).createWriter(fs, + HFile.Writer writer = + HFile.getWriterFactory(conf, cacheConf).createWriter(fs, hfilePath, SMALL_BLOCK_SIZE, compr, KeyValue.KEY_COMPARATOR); Random rand = new Random(19231737); @@ -505,8 +507,7 @@ } // Read the HFile - HFile.Reader reader = HFile.createReader(fs, hfilePath, blockCache, - false, true); + HFile.Reader reader = HFile.createReader(fs, hfilePath, cacheConf); assertEquals(expectedNumLevels, reader.getTrailer().getNumDataIndexLevels()); Index: src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java (working copy) @@ -45,7 +45,8 @@ Path ncTFile = new Path(this.testDir, "basic.hfile"); FSDataOutputStream fout = this.fs.create(ncTFile); int blocksize = toKV("a").getLength() * 3; - HFile.Writer writer = HFile.getWriterFactory(conf).createWriter(fout, + HFile.Writer writer = + HFile.getWriterFactory(conf).createWriter(fout, blocksize, "none", null); // 4 bytes * 3 * 2 for each key/value + // 3 for keys, 15 for values = 42 (woot) @@ -62,7 +63,7 @@ public void testSeekBefore() throws Exception { Path p = makeNewFile(); - HFile.Reader reader = HFile.createReader(fs, p, null, false, false); + HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf)); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, true); assertEquals(false, scanner.seekBefore(toKV("a").getKey())); @@ -95,7 +96,7 @@ public void testSeekTo() throws Exception { Path p = makeNewFile(); - HFile.Reader reader = HFile.createReader(fs, p, null, false, false); + HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf)); reader.loadFileInfo(); assertEquals(2, reader.getDataBlockIndexReader().getRootBlockCount()); HFileScanner scanner = reader.getScanner(false, true); @@ -115,7 +116,7 @@ public void testBlockContainingKey() throws Exception { Path p = makeNewFile(); - HFile.Reader reader = HFile.createReader(fs, p, null, false, false); + HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf)); reader.loadFileInfo(); HFileBlockIndex.BlockIndexReader blockIndexReader = reader.getDataBlockIndexReader(); Index: src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java (working copy) @@ -67,8 +67,8 @@ "testHFileFormatV2"); final Compression.Algorithm COMPRESS_ALGO = Compression.Algorithm.GZ; - HFileWriterV2 writer = new HFileWriterV2(conf, fs, hfilePath, 4096, - COMPRESS_ALGO, KeyValue.KEY_COMPARATOR); + HFileWriterV2 writer = new HFileWriterV2(conf, new CacheConfig(conf), fs, + hfilePath, 4096, COMPRESS_ALGO, KeyValue.KEY_COMPARATOR); long totalKeyLength = 0; long totalValueLength = 0; Index: src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java (revision 1182021) +++ src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java (working copy) @@ -20,23 +20,24 @@ package org.apache.hadoop.hbase.io; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Test; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import static org.junit.Assert.assertTrue; - - public class TestHalfStoreFileReader { /** @@ -63,8 +64,10 @@ Configuration conf = test_util.getConfiguration(); FileSystem fs = FileSystem.get(conf); + CacheConfig cacheConf = new CacheConfig(conf); - HFile.Writer w = HFile.getWriterFactory(conf).createWriter(fs, p, 1024, + HFile.Writer w = + HFile.getWriterFactory(conf, cacheConf).createWriter(fs, p, 1024, "none", KeyValue.KEY_COMPARATOR); // write some things. @@ -74,7 +77,7 @@ } w.close(); - HFile.Reader r = HFile.createReader(fs, p, null, false, false); + HFile.Reader r = HFile.createReader(fs, p, cacheConf); r.loadFileInfo(); byte [] midkey = r.midkey(); KeyValue midKV = KeyValue.createKeyValueFromKey(midkey); @@ -83,16 +86,17 @@ //System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey)); Reference bottom = new Reference(midkey, Reference.Range.bottom); - doTestOfScanAndReseek(p, fs, bottom); + doTestOfScanAndReseek(p, fs, bottom, cacheConf); Reference top = new Reference(midkey, Reference.Range.top); - doTestOfScanAndReseek(p, fs, top); + doTestOfScanAndReseek(p, fs, top, cacheConf); } - private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom) + private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom, + CacheConfig cacheConf) throws IOException { final HalfStoreFileReader halfreader = - new HalfStoreFileReader(fs, p, null, bottom); + new HalfStoreFileReader(fs, p, cacheConf, bottom); halfreader.loadFileInfo(); final HFileScanner scanner = halfreader.getScanner(false, false); Index: src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java (working copy) @@ -19,20 +19,19 @@ */ package org.apache.hadoop.hbase.util; +import java.io.IOException; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.Compression; import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.io.compress.Compressor; -import java.io.IOException; -import java.net.URI; - /** * Compression validation test. Checks compression is working. Be sure to run * on every node in your cluster. @@ -106,13 +105,14 @@ public static void doSmokeTest(FileSystem fs, Path path, String codec) throws Exception { Configuration conf = HBaseConfiguration.create(); - HFile.Writer writer = HFile.getWriterFactory(conf).createWriter( + HFile.Writer writer = + HFile.getWriterFactory(conf).createWriter( fs, path, HFile.DEFAULT_BLOCKSIZE, codec, null); writer.append(Bytes.toBytes("testkey"), Bytes.toBytes("testval")); writer.appendFileInfo(Bytes.toBytes("infokey"), Bytes.toBytes("infoval")); writer.close(); - HFile.Reader reader = HFile.createReader(fs, path, null, false, false); + HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf)); reader.loadFileInfo(); byte[] key = reader.getFirstKey(); boolean rc = Bytes.toString(key).equals("testkey"); Index: src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java (working copy) @@ -27,6 +27,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; @@ -75,10 +76,6 @@ public static final String IO_STOREFILE_BLOOM_BLOCK_SIZE = "io.storefile.bloom.block.size"; - /** Whether to cache compound Bloom filter blocks on write */ - public static final String IO_STOREFILE_BLOOM_CACHE_ON_WRITE = - "io.storefile.bloom.cacheonwrite"; - /** Maximum number of times a Bloom filter can be "folded" if oversized */ private static final int MAX_ALLOWED_FOLD_FACTOR = 7; @@ -140,7 +137,8 @@ * or when failed to create one. */ public static BloomFilterWriter createBloomAtWrite(Configuration conf, - BloomType bloomType, int maxKeys, HFile.Writer writer) { + CacheConfig cacheConf, BloomType bloomType, int maxKeys, + HFile.Writer writer) { if (!isBloomEnabled(conf)) { LOG.info("Bloom filters are disabled by configuration for " + writer.getPath() @@ -168,7 +166,7 @@ // In case of compound Bloom filters we ignore the maxKeys hint. CompoundBloomFilterWriter bloomWriter = new CompoundBloomFilterWriter( getBloomBlockSize(conf), err, Hash.getHashType(conf), maxFold, - cacheChunksOnWrite(conf), bloomType == BloomType.ROWCOL + cacheConf.shouldCacheBloomsOnWrite(), bloomType == BloomType.ROWCOL ? KeyValue.KEY_COMPARATOR : Bytes.BYTES_RAWCOMPARATOR); writer.addInlineBlockWriter(bloomWriter); return bloomWriter; @@ -199,10 +197,4 @@ public static int getBloomBlockSize(Configuration conf) { return conf.getInt(IO_STOREFILE_BLOOM_BLOCK_SIZE, 128 * 1024); } - - /** @return whether to cache compound Bloom filter chunks on write */ - public static boolean cacheChunksOnWrite(Configuration conf) { - return conf.getBoolean(IO_STOREFILE_BLOOM_CACHE_ON_WRITE, false); - } - }; Index: src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (working copy) @@ -58,7 +58,6 @@ import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HConstants.OperationStatusCode; import org.apache.hadoop.hbase.HDFSBlocksDistribution; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; @@ -75,6 +74,7 @@ import org.apache.hadoop.hbase.UnknownRowLockException; import org.apache.hadoop.hbase.UnknownScannerException; import org.apache.hadoop.hbase.YouAreDeadException; +import org.apache.hadoop.hbase.HConstants.OperationStatusCode; import org.apache.hadoop.hbase.catalog.CatalogTracker; import org.apache.hadoop.hbase.catalog.MetaEditor; import org.apache.hadoop.hbase.catalog.RootLocationEditor; @@ -96,10 +96,11 @@ import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType; import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.WritableByteArrayComparable; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; -import org.apache.hadoop.hbase.filter.WritableByteArrayComparable; import org.apache.hadoop.hbase.io.hfile.BlockCache; import org.apache.hadoop.hbase.io.hfile.BlockCacheColumnFamilySummary; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.CacheStats; import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; import org.apache.hadoop.hbase.ipc.HBaseRPC; @@ -289,6 +290,9 @@ private final RegionServerAccounting regionServerAccounting; + // Cache configuration and block cache reference + private final CacheConfig cacheConfig; + /** * The server name the Master sees us as. Its made from the hostname the * master passes us, port, and server startcode. Gets set after registration @@ -384,6 +388,7 @@ User.login(this.conf, "hbase.regionserver.keytab.file", "hbase.regionserver.kerberos.principal", this.isa.getHostName()); regionServerAccounting = new RegionServerAccounting(); + cacheConfig = new CacheConfig(conf); } /** @@ -685,9 +690,8 @@ } } // Send cache a shutdown. - BlockCache c = StoreFile.getBlockCache(this.conf); - if (c != null) { - c.shutdown(); + if (cacheConfig.isBlockCacheEnabled()) { + cacheConfig.getBlockCache().shutdown(); } // Send interrupts to wake up threads if sleeping so they notice shutdown. @@ -1279,7 +1283,7 @@ this.metrics.readRequestsCount.set(readRequestsCount); this.metrics.writeRequestsCount.set(writeRequestsCount); - BlockCache blockCache = StoreFile.getBlockCache(conf); + BlockCache blockCache = cacheConfig.getBlockCache(); if (blockCache != null) { this.metrics.blockCacheCount.set(blockCache.size()); this.metrics.blockCacheFree.set(blockCache.getFreeSize()); @@ -3149,7 +3153,7 @@ @Override public List getBlockCacheColumnFamilySummaries() throws IOException { - BlockCache c = StoreFile.getBlockCache(this.conf); + BlockCache c = new CacheConfig(this.conf).getBlockCache(); return c.getBlockCacheColumnFamilySummaries(this.conf); } Index: src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (working copy) @@ -85,6 +85,7 @@ import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; import org.apache.hadoop.hbase.ipc.HBaseRPC; import org.apache.hadoop.hbase.monitoring.MonitoredTask; @@ -4090,7 +4091,8 @@ processTable(fs, tableDir, log, c, majorCompact); } finally { log.close(); - BlockCache bc = StoreFile.getBlockCache(c); + // TODO: is this still right? + BlockCache bc = new CacheConfig(c).getBlockCache(); if (bc != null) bc.shutdown(); } } Index: src/main/java/org/apache/hadoop/hbase/regionserver/Store.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (working copy) @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.RemoteExceptionHandler; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.HeapSize; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.Compression; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileScanner; @@ -52,8 +53,8 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; +import org.apache.hadoop.hbase.util.CollectionBackedScanner; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.CollectionBackedScanner; import org.apache.hadoop.util.StringUtils; import com.google.common.base.Preconditions; @@ -93,6 +94,7 @@ private final HColumnDescriptor family; final FileSystem fs; final Configuration conf; + final CacheConfig cacheConf; // ttl in milliseconds. protected long ttl; protected int minVersions; @@ -115,7 +117,6 @@ private final Object flushLock = new Object(); final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); private final String storeNameStr; - private final boolean inMemory; private CompactionProgress progress; private final int compactionKVMax; @@ -195,8 +196,8 @@ conf.getInt("hbase.hstore.compaction.min", /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3))); - // Check if this is in-memory store - this.inMemory = family.isInMemory(); + // Setting up cache configuration for this family + this.cacheConf = new CacheConfig(conf, family); this.blockingStoreFileCount = conf.getInt("hbase.hstore.blockingStoreFiles", 7); @@ -270,8 +271,8 @@ } StoreFile curfile = null; try { - curfile = new StoreFile(fs, p, blockcache, this.conf, - this.family.getBloomFilterType(), this.inMemory); + curfile = new StoreFile(fs, p, this.conf, this.cacheConf, + this.family.getBloomFilterType()); curfile.createReader(); } catch (IOException ioe) { LOG.warn("Failed open of " + p + "; presumption is that file was " + @@ -335,7 +336,7 @@ LOG.info("Validating hfile at " + srcPath + " for inclusion in " + "store " + this + " region " + this.region); reader = HFile.createReader(srcPath.getFileSystem(conf), - srcPath, null, false, false); + srcPath, cacheConf); reader.loadFileInfo(); byte[] firstKey = reader.getFirstRowKey(); @@ -375,8 +376,8 @@ LOG.info("Renaming bulk load file " + srcPath + " to " + dstPath); StoreFile.rename(fs, srcPath, dstPath); - StoreFile sf = new StoreFile(fs, dstPath, blockcache, - this.conf, this.family.getBloomFilterType(), this.inMemory); + StoreFile sf = new StoreFile(fs, dstPath, this.conf, this.cacheConf, + this.family.getBloomFilterType()); sf.createReader(); LOG.info("Moved hfile " + srcPath + " into store directory " + @@ -530,8 +531,8 @@ } status.setStatus("Flushing " + this + ": reopening flushed file"); - StoreFile sf = new StoreFile(this.fs, dstPath, blockcache, - this.conf, this.family.getBloomFilterType(), this.inMemory); + StoreFile sf = new StoreFile(this.fs, dstPath, this.conf, this.cacheConf, + this.family.getBloomFilterType()); StoreFile.Reader r = sf.createReader(); this.storeSize += r.length(); this.totalUncompressedBytes += r.getTotalUncompressedBytes(); @@ -562,7 +563,7 @@ Compression.Algorithm compression) throws IOException { return StoreFile.createWriter(this.fs, region.getTmpDir(), this.blocksize, - compression, this.comparator, this.conf, + compression, this.comparator, this.conf, this.cacheConf, this.family.getBloomFilterType(), maxKeyCount); } @@ -1227,8 +1228,8 @@ LOG.error("Failed move of compacted file " + compactedFile.getPath(), e); return null; } - result = new StoreFile(this.fs, p, blockcache, this.conf, - this.family.getBloomFilterType(), this.inMemory); + result = new StoreFile(this.fs, p, this.conf, this.cacheConf, + this.family.getBloomFilterType()); result.createReader(); } this.lock.writeLock().lock(); @@ -1790,9 +1791,9 @@ } public static final long FIXED_OVERHEAD = ClassSize.align( - ClassSize.OBJECT + (16 * ClassSize.REFERENCE) + + ClassSize.OBJECT + (17 * ClassSize.REFERENCE) + (7 * Bytes.SIZEOF_LONG) + (1 * Bytes.SIZEOF_DOUBLE) + - (6 * Bytes.SIZEOF_INT) + (3 * Bytes.SIZEOF_BOOLEAN)); + (6 * Bytes.SIZEOF_INT) + (2 * Bytes.SIZEOF_BOOLEAN)); public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD + ClassSize.OBJECT + ClassSize.REENTRANT_LOCK + Index: src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (working copy) @@ -22,8 +22,6 @@ import java.io.DataInput; import java.io.FileNotFoundException; import java.io.IOException; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryUsage; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collection; @@ -49,22 +47,19 @@ import org.apache.hadoop.hbase.io.HalfStoreFileReader; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.Compression; -import org.apache.hadoop.hbase.io.hfile.DoubleBlockCache; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.io.hfile.HFileWriterV1; -import org.apache.hadoop.hbase.io.hfile.LruBlockCache; import org.apache.hadoop.hbase.util.BloomFilter; import org.apache.hadoop.hbase.util.BloomFilterFactory; import org.apache.hadoop.hbase.util.BloomFilterWriter; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.DirectMemoryUtils; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Writables; import org.apache.hadoop.io.RawComparator; import org.apache.hadoop.io.WritableUtils; -import org.apache.hadoop.util.StringUtils; import com.google.common.base.Function; import com.google.common.collect.ImmutableList; @@ -140,12 +135,9 @@ // If this StoreFile references another, this is the other files path. private Path referencePath; - // Should the block cache be used or not. - private boolean blockcache; + // Block cache configuration and reference. + private final CacheConfig cacheConf; - // Is this from an in-memory store - private boolean inMemory; - // HDFS blocks distribuion information private HDFSBlocksDistribution hdfsBlocksDistribution; @@ -200,6 +192,7 @@ * @param p The path of the file. * @param blockcache true if the block cache is enabled. * @param conf The current configuration. + * @param cacheConf The cache configuration and block cache reference. * @param cfBloomType The bloom type to use for this store file as specified * by column family configuration. This may or may not be the same * as the Bloom filter type actually present in the HFile, because @@ -209,16 +202,14 @@ */ StoreFile(final FileSystem fs, final Path p, - final boolean blockcache, final Configuration conf, - final BloomType cfBloomType, - final boolean inMemory) + final CacheConfig cacheConf, + final BloomType cfBloomType) throws IOException { this.conf = conf; this.fs = fs; this.path = p; - this.blockcache = blockcache; - this.inMemory = inMemory; + this.cacheConf = cacheConf; if (isReference(p)) { this.reference = Reference.read(fs, p); this.referencePath = getReferredToFile(this.path); @@ -366,49 +357,6 @@ } /** - * Returns the block cache or null in case none should be used. - * - * @param conf The current configuration. - * @return The block cache or null. - */ - public static synchronized BlockCache getBlockCache(Configuration conf) { - if (hfileBlockCache != null) return hfileBlockCache; - - float cachePercentage = conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY, 0.2f); - // There should be a better way to optimize this. But oh well. - if (cachePercentage == 0L) return null; - if (cachePercentage > 1.0) { - throw new IllegalArgumentException(HFILE_BLOCK_CACHE_SIZE_KEY + - " must be between 0.0 and 1.0, not > 1.0"); - } - - // Calculate the amount of heap to give the heap. - MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); - long cacheSize = (long)(mu.getMax() * cachePercentage); - int blockSize = conf.getInt("hbase.offheapcache.minblocksize", HFile.DEFAULT_BLOCKSIZE); - long offHeapCacheSize = (long) (conf.getFloat("hbase.offheapcache.percentage", (float) 0.95) * DirectMemoryUtils.getDirectMemorySize()); - boolean enableOffHeapCache = conf.getBoolean("hbase.offheapcache.enable", false); - LOG.info("Allocating LruBlockCache with maximum size " + - StringUtils.humanReadableInt(cacheSize)); - if(offHeapCacheSize <= 0 || !enableOffHeapCache) { - hfileBlockCache = new LruBlockCache(cacheSize, DEFAULT_BLOCKSIZE_SMALL); - } else { - LOG.info("Allocating OffHeapCache with maximum size " + - StringUtils.humanReadableInt(offHeapCacheSize)); - hfileBlockCache = new DoubleBlockCache(cacheSize, offHeapCacheSize, DEFAULT_BLOCKSIZE_SMALL, blockSize, conf); - } - return hfileBlockCache; - } - - /** - * @return the blockcache - */ - public BlockCache getBlockCache() { - return blockcache ? getBlockCache(conf) : null; - } - - - /** * @return the cached value of HDFS blocks distribution. The cached value is * calculated when store file is opened. */ @@ -497,11 +445,9 @@ } if (isReference()) { this.reader = new HalfStoreFileReader(this.fs, this.referencePath, - getBlockCache(), this.reference); + this.cacheConf, this.reference); } else { - this.reader = new Reader(this.fs, this.path, getBlockCache(), - this.inMemory, - this.conf.getBoolean(HFile.EVICT_BLOCKS_ON_CLOSE_KEY, true)); + this.reader = new Reader(this.fs, this.path, this.cacheConf); } computeHDFSBlockDistribution(); @@ -664,9 +610,10 @@ * @throws IOException */ public static Writer createWriter(final FileSystem fs, final Path dir, - final int blocksize, Configuration conf) throws IOException { - return createWriter(fs, dir, blocksize, null, null, conf, BloomType.NONE, - 0); + final int blocksize, Configuration conf, CacheConfig cacheConf) + throws IOException { + return createWriter(fs, dir, blocksize, null, null, conf, cacheConf, + BloomType.NONE, 0); } /** @@ -679,6 +626,7 @@ * @param algorithm Pass null to get default. * @param c Pass null to get default. * @param conf HBase system configuration. used with bloom filters + * @param cacheConf Cache configuration and reference. * @param bloomType column family setting for bloom filters * @param maxKeyCount estimated maximum number of keys we expect to add * @return HFile.Writer @@ -690,6 +638,7 @@ final Compression.Algorithm algorithm, final KeyValue.KVComparator c, final Configuration conf, + final CacheConfig cacheConf, BloomType bloomType, long maxKeyCount) throws IOException { @@ -704,7 +653,8 @@ return new Writer(fs, path, blocksize, algorithm == null? HFile.DEFAULT_COMPRESSION_ALGORITHM: algorithm, - conf, c == null ? KeyValue.COMPARATOR: c, bloomType, maxKeyCount); + conf, cacheConf, c == null ? KeyValue.COMPARATOR: c, bloomType, + maxKeyCount); } /** @@ -826,6 +776,7 @@ */ public Writer(FileSystem fs, Path path, int blocksize, Compression.Algorithm compress, final Configuration conf, + CacheConfig cacheConf, final KVComparator comparator, BloomType bloomType, long maxKeys) throws IOException { writer = HFile.getWriterFactory(conf).createWriter( @@ -834,7 +785,7 @@ this.kvComparator = comparator; - bloomFilterWriter = BloomFilterFactory.createBloomAtWrite(conf, + bloomFilterWriter = BloomFilterFactory.createBloomAtWrite(conf, cacheConf, bloomType, (int) Math.min(maxKeys, Integer.MAX_VALUE), writer); if (bloomFilterWriter != null) { this.bloomType = bloomType; @@ -1033,10 +984,9 @@ protected long sequenceID = -1; private byte[] lastBloomKey; - public Reader(FileSystem fs, Path path, BlockCache blockCache, - boolean inMemory, boolean evictOnClose) + public Reader(FileSystem fs, Path path, CacheConfig cacheConf) throws IOException { - reader = HFile.createReader(fs, path, blockCache, inMemory, evictOnClose); + reader = HFile.createReader(fs, path, cacheConf); bloomFilterType = BloomType.NONE; } Index: src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java (working copy) @@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.Compression; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.regionserver.StoreFile; @@ -169,7 +170,8 @@ Path familydir = new Path(outputdir, Bytes.toString(family)); String compression = compressionMap.get(family); compression = compression == null ? defaultCompression : compression; - wl.writer = HFile.getWriterFactory(conf).createWriter(fs, + wl.writer = + HFile.getWriterFactory(conf).createWriter(fs, StoreFile.getUniqueFile(fs, familydir), blocksize, compression, KeyValue.KEY_COMPARATOR); this.writers.put(family, wl); Index: src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java (working copy) @@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.io.HalfStoreFileReader; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.io.Reference.Range; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm; @@ -288,7 +289,8 @@ throws IOException { final Path hfilePath = item.hfilePath; final FileSystem fs = hfilePath.getFileSystem(getConf()); - HFile.Reader hfr = HFile.createReader(fs, hfilePath, null, false, false); + HFile.Reader hfr = HFile.createReader(fs, hfilePath, + new CacheConfig(getConf())); final byte[] first, last; try { hfr.loadFileInfo(); @@ -378,10 +380,12 @@ HColumnDescriptor familyDescriptor) throws IOException { FileSystem fs = inFile.getFileSystem(conf); + CacheConfig cacheConf = new CacheConfig(conf); HalfStoreFileReader halfReader = null; StoreFile.Writer halfWriter = null; try { - halfReader = new HalfStoreFileReader(fs, inFile, null, reference); + halfReader = new HalfStoreFileReader(fs, inFile, cacheConf, + reference); Map fileInfo = halfReader.loadFileInfo(); int blocksize = familyDescriptor.getBlocksize(); @@ -389,8 +393,8 @@ BloomType bloomFilterType = familyDescriptor.getBloomFilterType(); halfWriter = new StoreFile.Writer( - fs, outFile, blocksize, compression, conf, KeyValue.COMPARATOR, - bloomFilterType, 0); + fs, outFile, blocksize, compression, conf, cacheConf, + KeyValue.COMPARATOR, bloomFilterType, 0); HFileScanner scanner = halfReader.getScanner(false, false); scanner.seekTo(); do { @@ -490,7 +494,8 @@ for (Path hfile : hfiles) { if (hfile.getName().startsWith("_")) continue; - HFile.Reader reader = HFile.createReader(fs, hfile, null, false, false); + HFile.Reader reader = HFile.createReader(fs, hfile, + new CacheConfig(getConf())); final byte[] first, last; try { reader.loadFileInfo(); Index: src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (working copy) @@ -40,7 +40,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; @@ -525,6 +524,11 @@ return this.elements.get(); } + @Override + public long getBlockCount() { + return this.elements.get(); + } + /** * Get the number of eviction runs that have occurred */ @@ -624,7 +628,7 @@ public CacheStats getStats() { return this.stats; } - + public final static long CACHE_FIXED_OVERHEAD = ClassSize.align( (3 * Bytes.SIZEOF_LONG) + (8 * ClassSize.REFERENCE) + (5 * Bytes.SIZEOF_FLOAT) + Bytes.SIZEOF_BOOLEAN @@ -645,18 +649,18 @@ @Override public List getBlockCacheColumnFamilySummaries(Configuration conf) throws IOException { - + Map sfMap = FSUtils.getTableStoreFilePathMap( FileSystem.get(conf), FSUtils.getRootDir(conf)); - - // quirky, but it's a compound key and this is a shortcut taken instead of + + // quirky, but it's a compound key and this is a shortcut taken instead of // creating a class that would represent only a key. - Map bcs = + Map bcs = new HashMap(); final String pattern = "\\" + HFile.CACHE_KEY_SEPARATOR; - + for (CachedBlock cb : map.values()) { // split name and get the first part (e.g., "8351478435190657655_0") // see HFile.getBlockCacheKey for structure of block cache key. @@ -665,7 +669,7 @@ String sf = s[0]; Path path = sfMap.get(sf); if ( path != null) { - BlockCacheColumnFamilySummary lookup = + BlockCacheColumnFamilySummary lookup = BlockCacheColumnFamilySummary.createFromStoreFilePath(path); BlockCacheColumnFamilySummary bcse = bcs.get(lookup); if (bcse == null) { @@ -677,12 +681,12 @@ } } } - List list = + List list = new ArrayList(bcs.values()); - Collections.sort( list ); + Collections.sort( list ); return list; } - + // Simple calculators of sizes given factors and maxSize private long acceptableSize() { Index: src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java (working copy) @@ -269,6 +269,11 @@ return this.stats; } + @Override + public long getBlockCount() { + return numBlocks - backingStore.getBlocksRemaining(); + } + /* Since its offheap, it doesn't matter if its in memory or not */ @Override public void cacheBlock(String blockName, Cacheable buf, boolean inMemory) { Index: src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java (working copy) @@ -375,6 +375,15 @@ return 0; // this cache, by default, allocates all its space. } + @Override + public long getBlockCount() { + long count = 0; + for (SingleSizeCache cache : backingStore.values()) { + count += cache.getBlockCount(); + } + return count; + } + public long getCurrentSize() { return size; } Index: src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java (working copy) @@ -185,7 +185,7 @@ System.err.println("ERROR, file doesnt exist: " + file); } - HFile.Reader reader = HFile.createReader(fs, file, null, false, false); + HFile.Reader reader = HFile.createReader(fs, file, new CacheConfig(conf)); Map fileInfo = reader.loadFileInfo(); Index: src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java (working copy) @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hbase.KeyValue.KeyComparator; import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; -import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.RawComparator; import org.apache.hadoop.io.Writable; @@ -91,25 +90,16 @@ /** May be null if we were passed a stream. */ protected final Path path; - /** Whether to cache key/value data blocks on write */ - protected final boolean cacheDataBlocksOnWrite; + /** Cache configuration for caching data on write. */ + protected final CacheConfig cacheConf; - /** Whether to cache non-root index blocks on write */ - protected final boolean cacheIndexBlocksOnWrite; - - /** Block cache to optionally fill on write. */ - protected BlockCache blockCache; - - /** Configuration used for block cache initialization */ - private Configuration conf; - /** * Name for this object used when logging or in toString. Is either * the result of a toString on stream or else toString of passed file Path. */ protected final String name; - public AbstractHFileWriter(Configuration conf, + public AbstractHFileWriter(CacheConfig cacheConf, FSDataOutputStream outputStream, Path path, int blockSize, Compression.Algorithm compressAlgo, KeyComparator comparator) { this.outputStream = outputStream; @@ -122,15 +112,7 @@ : Bytes.BYTES_RAWCOMPARATOR; closeOutputStream = path != null; - - cacheDataBlocksOnWrite = conf.getBoolean(HFile.CACHE_BLOCKS_ON_WRITE_KEY, - false); - cacheIndexBlocksOnWrite = HFileBlockIndex.shouldCacheOnWrite(conf); - - this.conf = conf; - - if (cacheDataBlocksOnWrite || cacheIndexBlocksOnWrite) - initBlockCache(); + this.cacheConf = cacheConf; } /** @@ -275,13 +257,4 @@ fs.getDefaultReplication(), fs.getDefaultBlockSize(), null); } - - /** Initializes the block cache to use for cache-on-write */ - protected void initBlockCache() { - if (blockCache == null) { - blockCache = StoreFile.getBlockCache(conf); - conf = null; // This is all we need configuration for. - } - } - } Index: src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java (working copy) @@ -166,4 +166,9 @@ return onHeapCache.getBlockCacheColumnFamilySummaries(conf); } + @Override + public long getBlockCount() { + return onHeapCache.getBlockCount() + offHeapCache.getBlockCount(); + } + } Index: src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java (working copy) @@ -83,15 +83,37 @@ */ public void shutdown(); + /** + * Returns the total size of the block cache, in bytes. + * @return size of cache, in bytes + */ public long size(); + /** + * Returns the free size of the block cache, in bytes. + * @return free space in cache, in bytes + */ public long getFreeSize(); + /** + * Returns the occupied size of the block cache, in bytes. + * @return occupied space in cache, in bytes + */ public long getCurrentSize(); + /** + * Returns the number of evictions that have occurred. + * @return number of evictions + */ public long getEvictedCount(); /** + * Returns the number of blocks currently cached in the block cache. + * @return number of blocks in the cache + */ + public long getBlockCount(); + + /** * Performs a BlockCache summary and returns a List of BlockCacheColumnFamilySummary objects. * This method could be fairly heavyweight in that it evaluates the entire HBase file-system * against what is in the RegionServer BlockCache. Index: src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java (working copy) @@ -54,18 +54,14 @@ * @param fsdis input stream. Caller is responsible for closing the passed * stream. * @param size Length of the stream. - * @param blockCache block cache. Pass null if none. - * @param inMemory whether blocks should be marked as in-memory in cache - * @param evictOnClose whether blocks in cache should be evicted on close + * @param cacheConf cache references and configuration * @throws IOException */ public HFileReaderV1(Path path, FixedFileTrailer trailer, final FSDataInputStream fsdis, final long size, final boolean closeIStream, - final BlockCache blockCache, final boolean inMemory, - final boolean evictOnClose) { - super(path, trailer, fsdis, size, closeIStream, blockCache, inMemory, - evictOnClose); + final CacheConfig cacheConf) { + super(path, trailer, fsdis, size, closeIStream, cacheConf); trailer.expectVersion(1); fsBlockReader = new HFileBlock.FSReaderV1(fsdis, compressAlgo, fileSize); @@ -221,9 +217,10 @@ synchronized (metaBlockIndexReader.getRootBlockKey(block)) { metaLoads.incrementAndGet(); // Check cache for block. If found return. - if (blockCache != null) { - HFileBlock cachedBlock = (HFileBlock) blockCache.getBlock(cacheKey, - true); + if (cacheConf.isBlockCacheEnabled()) { + HFileBlock cachedBlock = + (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey, + cacheConf.shouldCacheDataOnRead()); if (cachedBlock != null) { cacheHits.incrementAndGet(); return cachedBlock.getBufferWithoutHeader(); @@ -240,8 +237,9 @@ HFile.readOps.incrementAndGet(); // Cache the block - if (cacheBlock && blockCache != null) { - blockCache.cacheBlock(cacheKey, hfileBlock, inMemory); + if (cacheConf.shouldCacheDataOnRead() && cacheBlock) { + cacheConf.getBlockCache().cacheBlock(cacheKey, hfileBlock, + cacheConf.isInMemory()); } return hfileBlock.getBufferWithoutHeader(); @@ -279,9 +277,10 @@ blockLoads.incrementAndGet(); // Check cache for block. If found return. - if (blockCache != null) { - HFileBlock cachedBlock = (HFileBlock) blockCache.getBlock(cacheKey, - true); + if (cacheConf.isBlockCacheEnabled()) { + HFileBlock cachedBlock = + (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey, + cacheConf.shouldCacheDataOnRead()); if (cachedBlock != null) { cacheHits.incrementAndGet(); return cachedBlock.getBufferWithoutHeader(); @@ -312,8 +311,9 @@ HFile.readOps.incrementAndGet(); // Cache the block - if (cacheBlock && blockCache != null) { - blockCache.cacheBlock(cacheKey, hfileBlock, inMemory); + if (cacheConf.shouldCacheDataOnRead() && cacheBlock) { + cacheConf.getBlockCache().cacheBlock(cacheKey, hfileBlock, + cacheConf.isInMemory()); } return buf; @@ -348,10 +348,10 @@ @Override public void close() throws IOException { - if (evictOnClose && this.blockCache != null) { + if (cacheConf.shouldEvictOnClose()) { int numEvicted = 0; for (int i = 0; i < dataBlockIndexReader.getRootBlockCount(); i++) { - if (blockCache.evictBlock(HFile.getBlockCacheKey(name, + if (cacheConf.getBlockCache().evictBlock(HFile.getBlockCacheKey(name, dataBlockIndexReader.getRootBlockOffset(i)))) numEvicted++; } Index: src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java (working copy) @@ -65,20 +65,20 @@ * Opens a HFile. You must load the index before you can use it by calling * {@link #loadFileInfo()}. * + * @param path Path to HFile. + * @param trailer File trailer. * @param fsdis input stream. Caller is responsible for closing the passed * stream. * @param size Length of the stream. - * @param blockCache block cache. Pass null if none. - * @param inMemory whether blocks should be marked as in-memory in cache - * @param evictOnClose whether blocks in cache should be evicted on close + * @param closeIStream Whether to close the stream. + * @param cacheConf Cache configuration. * @throws IOException */ public HFileReaderV2(Path path, FixedFileTrailer trailer, final FSDataInputStream fsdis, final long size, - final boolean closeIStream, final BlockCache blockCache, - final boolean inMemory, final boolean evictOnClose) throws IOException { - super(path, trailer, fsdis, size, closeIStream, blockCache, inMemory, - evictOnClose); + final boolean closeIStream, final CacheConfig cacheConf) + throws IOException { + super(path, trailer, fsdis, size, closeIStream, cacheConf); trailer.expectVersion(2); fsBlockReader = new HFileBlock.FSReaderV2(fsdis, compressAlgo, @@ -174,9 +174,10 @@ long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block); String cacheKey = HFile.getBlockCacheKey(name, metaBlockOffset); - if (blockCache != null) { - HFileBlock cachedBlock = (HFileBlock) blockCache.getBlock(cacheKey, - true); + cacheBlock &= cacheConf.shouldCacheDataOnRead(); + if (cacheConf.isBlockCacheEnabled()) { + HFileBlock cachedBlock = + (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock); if (cachedBlock != null) { // Return a distinct 'shallow copy' of the block, // so pos does not get messed by the scanner @@ -193,8 +194,9 @@ HFile.readOps.incrementAndGet(); // Cache the block - if (cacheBlock && blockCache != null) { - blockCache.cacheBlock(cacheKey, metaBlock, inMemory); + if (cacheBlock) { + cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock, + cacheConf.isInMemory()); } return metaBlock.getBufferWithoutHeader(); @@ -237,9 +239,10 @@ blockLoads.incrementAndGet(); // Check cache for block. If found return. - if (blockCache != null) { - HFileBlock cachedBlock = (HFileBlock) blockCache.getBlock(cacheKey, - true); + cacheBlock &= cacheConf.shouldCacheDataOnRead(); + if (cacheConf.isBlockCacheEnabled()) { + HFileBlock cachedBlock = + (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock); if (cachedBlock != null) { cacheHits.incrementAndGet(); @@ -257,8 +260,9 @@ HFile.readOps.incrementAndGet(); // Cache the block - if (cacheBlock && blockCache != null) { - blockCache.cacheBlock(cacheKey, dataBlock, inMemory); + if (cacheBlock) { + cacheConf.getBlockCache().cacheBlock(cacheKey, dataBlock, + cacheConf.isInMemory()); } return dataBlock; @@ -289,8 +293,8 @@ @Override public void close() throws IOException { - if (evictOnClose && blockCache != null) { - int numEvicted = blockCache.evictBlocksByPrefix(name + if (cacheConf.shouldEvictOnClose()) { + int numEvicted = cacheConf.getBlockCache().evictBlocksByPrefix(name + HFile.CACHE_KEY_SEPARATOR); LOG.debug("On close of file " + name + " evicted " + numEvicted + " block(s)"); Index: src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (working copy) @@ -198,8 +198,12 @@ */ public static abstract class WriterFactory { protected Configuration conf; + protected CacheConfig cacheConf; - WriterFactory(Configuration conf) { this.conf = conf; } + WriterFactory(Configuration conf, CacheConfig cacheConf) { + this.conf = conf; + this.cacheConf = cacheConf; + } public abstract Writer createWriter(FileSystem fs, Path path) throws IOException; @@ -236,33 +240,29 @@ * can also be {@link HFileWriterV1#WRITER_FACTORY_V1} in testing. */ public static final WriterFactory getWriterFactory(Configuration conf) { + return HFile.getWriterFactory(conf, new CacheConfig(conf)); + } + + /** + * Returns the factory to be used to create {@link HFile} writers. Should + * always be {@link HFileWriterV2#WRITER_FACTORY_V2} in production, but + * can also be {@link HFileWriterV1#WRITER_FACTORY_V1} in testing. + */ + public static final WriterFactory getWriterFactory(Configuration conf, + CacheConfig cacheConf) { int version = getFormatVersion(conf); LOG.debug("Using HFile format version " + version); switch (version) { case 1: - return new HFileWriterV1.WriterFactoryV1(conf); + return new HFileWriterV1.WriterFactoryV1(conf, cacheConf); case 2: - return new HFileWriterV2.WriterFactoryV2(conf); + return new HFileWriterV2.WriterFactoryV2(conf, cacheConf); default: throw new IllegalArgumentException("Cannot create writer for HFile " + "format version " + version); } } - /** - * Configuration key to evict all blocks of a given file from the block cache - * when the file is closed. - */ - public static final String EVICT_BLOCKS_ON_CLOSE_KEY = - "hbase.rs.evictblocksonclose"; - - /** - * Configuration key to cache data blocks on write. There are separate - * switches for Bloom blocks and non-root index blocks. - */ - public static final String CACHE_BLOCKS_ON_WRITE_KEY = - "hbase.rs.cacheblocksonwrite"; - /** An abstraction used by the block index */ public interface CachingBlockReader { HFileBlock readBlock(long offset, long onDiskBlockSize, @@ -325,35 +325,32 @@ } private static Reader pickReaderVersion(Path path, FSDataInputStream fsdis, - long size, boolean closeIStream, BlockCache blockCache, - boolean inMemory, boolean evictOnClose) throws IOException { + long size, boolean closeIStream, CacheConfig cacheConf) + throws IOException { FixedFileTrailer trailer = FixedFileTrailer.readFromStream(fsdis, size); switch (trailer.getVersion()) { case 1: return new HFileReaderV1(path, trailer, fsdis, size, closeIStream, - blockCache, inMemory, evictOnClose); + cacheConf); case 2: return new HFileReaderV2(path, trailer, fsdis, size, closeIStream, - blockCache, inMemory, evictOnClose); + cacheConf); default: throw new IOException("Cannot instantiate reader for HFile version " + trailer.getVersion()); } } - public static Reader createReader( - FileSystem fs, Path path, BlockCache blockCache, boolean inMemory, - boolean evictOnClose) throws IOException { + public static Reader createReader(FileSystem fs, Path path, + CacheConfig cacheConf) throws IOException { return pickReaderVersion(path, fs.open(path), - fs.getFileStatus(path).getLen(), true, blockCache, inMemory, - evictOnClose); + fs.getFileStatus(path).getLen(), true, cacheConf); } public static Reader createReader(Path path, FSDataInputStream fsdis, - long size, BlockCache blockache, boolean inMemory, boolean evictOnClose) + long size, CacheConfig cacheConf) throws IOException { - return pickReaderVersion(path, fsdis, size, false, blockache, inMemory, - evictOnClose); + return pickReaderVersion(path, fsdis, size, false, cacheConf); } /* Index: src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java (working copy) @@ -128,5 +128,11 @@ throw new UnsupportedOperationException(); } + @Override + public long getBlockCount() { + // TODO: implement this if we ever actually use this block cache + return 0; + } + } Index: src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java (revision 0) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java (revision 0) @@ -0,0 +1,326 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryUsage; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.util.DirectMemoryUtils; +import org.apache.hadoop.util.StringUtils; + +/** + * Stores all of the cache objects and configuration for a single HFile. + */ +public class CacheConfig { + private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName()); + + /** + * Configuration key for the size of the block cache, in bytes. + */ + public static final String HFILE_BLOCK_CACHE_SIZE_KEY = + "hfile.block.cache.size"; + + /** + * Configuration key to cache data blocks on write. There are separate + * switches for bloom blocks and non-root index blocks. + */ + public static final String CACHE_BLOCKS_ON_WRITE_KEY = + "hbase.rs.cacheblocksonwrite"; + + /** + * Configuration key to cache leaf and intermediate-level index blocks on + * write. + */ + public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY = + "hfile.block.index.cacheonwrite"; + + /** + * Configuration key to cache compound bloom filter blocks on write. + */ + public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY = + "hfile.block.bloom.cacheonwrite"; + + /** + * TODO: Implement this (jgray) + * Configuration key to cache data blocks in compressed format. + */ + public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY = + "hbase.rs.blockcache.cachedatacompressed"; + + /** + * Configuration key to evict all blocks of a given file from the block cache + * when the file is closed. + */ + public static final String EVICT_BLOCKS_ON_CLOSE_KEY = + "hbase.rs.evictblocksonclose"; + + // Defaults + + public static final boolean DEFAULT_CACHE_DATA_ON_READ = true; + public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false; + public static final boolean DEFAULT_IN_MEMORY = false; + public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false; + public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false; + public static final boolean DEFAULT_EVICT_ON_CLOSE = false; + public static final boolean DEFAULT_COMPRESSED_CACHE = false; + + /** Local reference to the block cache, null if completely disabled */ + private final BlockCache blockCache; + + /** + * Whether blocks should be cached on read (default is on if there is a + * cache but this can be turned off on a per-family or per-request basis) + */ + private boolean cacheDataOnRead; + + /** Whether blocks should be flagged as in-memory when being cached */ + private final boolean inMemory; + + /** Whether data blocks should be cached when new files are written */ + private final boolean cacheDataOnWrite; + + /** Whether index blocks should be cached when new files are written */ + private final boolean cacheIndexesOnWrite; + + /** Whether compound bloom filter blocks should be cached on write */ + private final boolean cacheBloomsOnWrite; + + /** Whether blocks of a file should be evicted when the file is closed */ + private final boolean evictOnClose; + + /** Whether data blocks should be stored in compressed form in the cache */ + private final boolean cacheCompressed; + + /** + * Create a cache configuration using the specified configuration object and + * family descriptor. + * @param conf hbase configuration + * @param family column family configuration + */ + public CacheConfig(Configuration conf, HColumnDescriptor family) { + this(CacheConfig.instantiateBlockCache(conf), + family.isBlockCacheEnabled(), family.isInMemory(), + conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE), + conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, + DEFAULT_CACHE_INDEXES_ON_WRITE), + conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, + DEFAULT_CACHE_BLOOMS_ON_WRITE), + conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE), + conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_COMPRESSED_CACHE) + ); + } + + /** + * Create a cache configuration using the specified configuration object and + * defaults for family level settings. + * @param conf hbase configuration + */ + public CacheConfig(Configuration conf) { + this(CacheConfig.instantiateBlockCache(conf), + DEFAULT_CACHE_DATA_ON_READ, + DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set + // strictly from conf + conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE), + conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, + DEFAULT_CACHE_INDEXES_ON_WRITE), + conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, + DEFAULT_CACHE_BLOOMS_ON_WRITE), + conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE), + conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, + DEFAULT_COMPRESSED_CACHE) + ); + } + + /** + * Create a block cache configuration with the specified cache and + * configuration parameters. + * @param blockCache reference to block cache, null if completely disabled + * @param cacheDataOnRead whether data blocks should be cached on read + * @param inMemory whether blocks should be flagged as in-memory + * @param cacheDataOnWrite whether data blocks should be cached on write + * @param cacheIndexesOnWrite whether index blocks should be cached on write + * @param cacheBloomsOnWrite whether blooms should be cached on write + * @param evictOnClose whether blocks should be evicted when HFile is closed + * @param cacheCompressed whether to store blocks as compressed in the cache + */ + CacheConfig(final BlockCache blockCache, + final boolean cacheDataOnRead, final boolean inMemory, + final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite, + final boolean cacheBloomsOnWrite, final boolean evictOnClose, + final boolean cacheCompressed) { + this.blockCache = blockCache; + this.cacheDataOnRead = cacheDataOnRead; + this.inMemory = inMemory; + this.cacheDataOnWrite = cacheDataOnWrite; + this.cacheIndexesOnWrite = cacheIndexesOnWrite; + this.cacheBloomsOnWrite = cacheBloomsOnWrite; + this.evictOnClose = evictOnClose; + this.cacheCompressed = cacheCompressed; + } + + /** + * Constructs a cache configuration copied from the specified configuration. + * @param cacheConf + */ + public CacheConfig(CacheConfig cacheConf) { + this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory, + cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite, + cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose, + cacheConf.cacheCompressed); + } + + /** + * Checks whether the block cache is enabled. + */ + public boolean isBlockCacheEnabled() { + return this.blockCache != null; + } + + /** + * Returns the block cache. + * @return the block cache, or null if caching is completely disabled + */ + public BlockCache getBlockCache() { + return this.blockCache; + } + + /** + * Returns whether the blocks of this HFile should be cached on read or not. + * @return true if blocks should be cached on read, false if not + */ + public boolean shouldCacheDataOnRead() { + return isBlockCacheEnabled() && cacheDataOnRead; + } + + /** + * @return true if blocks in this file should be flagged as in-memory + */ + public boolean isInMemory() { + return isBlockCacheEnabled() && this.inMemory; + } + + /** + * @return true if data blocks should be written to the cache when an HFile is + * written, false if not + */ + public boolean shouldCacheDataOnWrite() { + return isBlockCacheEnabled() && this.cacheDataOnWrite; + } + + /** + * @return true if index blocks should be written to the cache when an HFile + * is written, false if not + */ + public boolean shouldCacheIndexesOnWrite() { + return isBlockCacheEnabled() && this.cacheIndexesOnWrite; + } + + /** + * @return true if bloom blocks should be written to the cache when an HFile + * is written, false if not + */ + public boolean shouldCacheBloomsOnWrite() { + return isBlockCacheEnabled() && this.cacheBloomsOnWrite; + } + + /** + * @return true if blocks should be evicted from the cache when an HFile + * reader is closed, false if not + */ + public boolean shouldEvictOnClose() { + return isBlockCacheEnabled() && this.evictOnClose; + } + + /** + * @return true if blocks should be compressed in the cache, false if not + */ + public boolean shouldCacheCompressed() { + return isBlockCacheEnabled() && this.cacheCompressed; + } + + @Override + public String toString() { + if (!isBlockCacheEnabled()) { + return "CacheConfig:disabled"; + } + return "CacheConfig:enabled " + + "[cacheDataOnRead=" + shouldCacheDataOnRead() + "] " + + "[cacheDataOnWrite=" + shouldCacheDataOnWrite() + "] " + + "[cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + "] " + + "[cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + "] " + + "[cacheEvictOnClose=" + shouldEvictOnClose() + "] " + + "[cacheCompressed=" + shouldCacheCompressed() + "]"; + } + + // Static block cache reference and methods + + /** + * Static reference to the block cache, or null if no caching should be used + * at all. + */ + private static BlockCache globalBlockCache; + + /** Boolean whether we have disabled the block cache entirely. */ + private static boolean blockCacheDisabled = false; + + /** + * Returns the block cache or null in case none should be used. + * + * @param conf The current configuration. + * @return The block cache or null. + */ + private static synchronized BlockCache instantiateBlockCache( + Configuration conf) { + if (globalBlockCache != null) return globalBlockCache; + if (blockCacheDisabled) return null; + + float cachePercentage = conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY, 0.2f); + if (cachePercentage == 0L) { + blockCacheDisabled = true; + return null; + } + if (cachePercentage > 1.0) { + throw new IllegalArgumentException(HFILE_BLOCK_CACHE_SIZE_KEY + + " must be between 0.0 and 1.0, not > 1.0"); + } + + // Calculate the amount of heap to give the heap. + MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); + long cacheSize = (long)(mu.getMax() * cachePercentage); + int blockSize = conf.getInt("hbase.offheapcache.minblocksize", + HFile.DEFAULT_BLOCKSIZE); + long offHeapCacheSize = + (long) (conf.getFloat("hbase.offheapcache.percentage", (float) 0.95) * + DirectMemoryUtils.getDirectMemorySize()); + LOG.info("Allocating LruBlockCache with maximum size " + + StringUtils.humanReadableInt(cacheSize)); + if (offHeapCacheSize <= 0) { + globalBlockCache = new LruBlockCache(cacheSize, + StoreFile.DEFAULT_BLOCKSIZE_SMALL); + } else { + globalBlockCache = new DoubleBlockCache(cacheSize, offHeapCacheSize, + StoreFile.DEFAULT_BLOCKSIZE_SMALL, blockSize, conf); + } + return globalBlockCache; + } +} Index: src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java (working copy) @@ -79,18 +79,20 @@ static class WriterFactoryV1 extends HFile.WriterFactory { - WriterFactoryV1(Configuration conf) { super(conf); } + WriterFactoryV1(Configuration conf, CacheConfig cacheConf) { + super(conf, cacheConf); + } @Override public Writer createWriter(FileSystem fs, Path path) throws IOException { - return new HFileWriterV1(conf, fs, path); + return new HFileWriterV1(conf, cacheConf, fs, path); } @Override public Writer createWriter(FileSystem fs, Path path, int blockSize, Compression.Algorithm compressAlgo, final KeyComparator comparator) throws IOException { - return new HFileWriterV1(conf, fs, path, blockSize, + return new HFileWriterV1(conf, cacheConf, fs, path, blockSize, compressAlgo, comparator); } @@ -98,7 +100,7 @@ public Writer createWriter(FileSystem fs, Path path, int blockSize, String compressAlgoName, final KeyComparator comparator) throws IOException { - return new HFileWriterV1(conf, fs, path, blockSize, + return new HFileWriterV1(conf, cacheConf, fs, path, blockSize, compressAlgoName, comparator); } @@ -106,21 +108,23 @@ public Writer createWriter(final FSDataOutputStream ostream, final int blockSize, final String compress, final KeyComparator comparator) throws IOException { - return new HFileWriterV1(conf, ostream, blockSize, compress, comparator); + return new HFileWriterV1(cacheConf, ostream, blockSize, compress, + comparator); } @Override public Writer createWriter(final FSDataOutputStream ostream, final int blockSize, final Compression.Algorithm compress, final KeyComparator c) throws IOException { - return new HFileWriterV1(conf, ostream, blockSize, compress, c); + return new HFileWriterV1(cacheConf, ostream, blockSize, compress, c); } } /** Constructor that uses all defaults for compression and block size. */ - public HFileWriterV1(Configuration conf, FileSystem fs, Path path) + public HFileWriterV1(Configuration conf, CacheConfig cacheConf, + FileSystem fs, Path path) throws IOException { - this(conf, fs, path, HFile.DEFAULT_BLOCKSIZE, + this(conf, cacheConf, fs, path, HFile.DEFAULT_BLOCKSIZE, HFile.DEFAULT_COMPRESSION_ALGORITHM, null); } @@ -129,37 +133,37 @@ * Constructor that takes a path, creates and closes the output stream. Takes * compression algorithm name as string. */ - public HFileWriterV1(Configuration conf, FileSystem fs, Path path, - int blockSize, String compressAlgoName, + public HFileWriterV1(Configuration conf, CacheConfig cacheConf, FileSystem fs, + Path path, int blockSize, String compressAlgoName, final KeyComparator comparator) throws IOException { - this(conf, fs, path, blockSize, + this(conf, cacheConf, fs, path, blockSize, compressionByName(compressAlgoName), comparator); } /** Constructor that takes a path, creates and closes the output stream. */ - public HFileWriterV1(Configuration conf, FileSystem fs, Path path, - int blockSize, Compression.Algorithm compress, + public HFileWriterV1(Configuration conf, CacheConfig cacheConf, FileSystem fs, + Path path, int blockSize, Compression.Algorithm compress, final KeyComparator comparator) throws IOException { - super(conf, createOutputStream(conf, fs, path), path, + super(cacheConf, createOutputStream(conf, fs, path), path, blockSize, compress, comparator); } /** Constructor that takes a stream. */ - public HFileWriterV1(Configuration conf, + public HFileWriterV1(CacheConfig cacheConf, final FSDataOutputStream outputStream, final int blockSize, final String compressAlgoName, final KeyComparator comparator) throws IOException { - this(conf, outputStream, blockSize, + this(cacheConf, outputStream, blockSize, Compression.getCompressionAlgorithmByName(compressAlgoName), comparator); } /** Constructor that takes a stream. */ - public HFileWriterV1(Configuration conf, + public HFileWriterV1(CacheConfig cacheConf, final FSDataOutputStream outputStream, final int blockSize, final Compression.Algorithm compress, final KeyComparator comparator) throws IOException { - super(conf, outputStream, null, blockSize, compress, comparator); + super(cacheConf, outputStream, null, blockSize, compress, comparator); } /** @@ -194,10 +198,11 @@ HFile.writeTimeNano.addAndGet(System.nanoTime() - startTimeNs); HFile.writeOps.incrementAndGet(); - if (cacheDataBlocksOnWrite) { + if (cacheConf.shouldCacheDataOnWrite()) { baosDos.flush(); byte[] bytes = baos.toByteArray(); - blockCache.cacheBlock(HFile.getBlockCacheKey(name, blockBegin), + cacheConf.getBlockCache().cacheBlock( + HFile.getBlockCacheKey(name, blockBegin), new HFileBlock(BlockType.DATA, (int) (outputStream.getPos() - blockBegin), bytes.length, -1, ByteBuffer.wrap(bytes, 0, bytes.length), true, blockBegin)); @@ -217,7 +222,7 @@ this.out = getCompressingStream(); BlockType.DATA.write(out); firstKeyInBlock = null; - if (cacheDataBlocksOnWrite) { + if (cacheConf.shouldCacheDataOnWrite()) { this.baos = new ByteArrayOutputStream(); this.baosDos = new DataOutputStream(baos); baosDos.write(HFileBlock.DUMMY_HEADER); @@ -361,7 +366,7 @@ this.lastKeyLength = klength; this.entryCount++; // If we are pre-caching blocks on write, fill byte array stream - if (cacheDataBlocksOnWrite) { + if (cacheConf.shouldCacheDataOnWrite()) { this.baosDos.writeInt(klength); this.baosDos.writeInt(vlength); this.baosDos.write(key, koffset, klength); Index: src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java (working copy) @@ -65,19 +65,21 @@ static class WriterFactoryV2 extends HFile.WriterFactory { - WriterFactoryV2(Configuration conf) { super(conf); } + WriterFactoryV2(Configuration conf, CacheConfig cacheConf) { + super(conf, cacheConf); + } @Override public Writer createWriter(FileSystem fs, Path path) throws IOException { - return new HFileWriterV2(conf, fs, path); + return new HFileWriterV2(conf, cacheConf, fs, path); } @Override public Writer createWriter(FileSystem fs, Path path, int blockSize, Compression.Algorithm compress, final KeyComparator comparator) throws IOException { - return new HFileWriterV2(conf, fs, path, blockSize, + return new HFileWriterV2(conf, cacheConf, fs, path, blockSize, compress, comparator); } @@ -85,7 +87,7 @@ public Writer createWriter(FileSystem fs, Path path, int blockSize, String compress, final KeyComparator comparator) throws IOException { - return new HFileWriterV2(conf, fs, path, blockSize, + return new HFileWriterV2(conf, cacheConf, fs, path, blockSize, compress, comparator); } @@ -93,21 +95,24 @@ public Writer createWriter(final FSDataOutputStream ostream, final int blockSize, final String compress, final KeyComparator comparator) throws IOException { - return new HFileWriterV2(conf, ostream, blockSize, compress, comparator); + return new HFileWriterV2(conf, cacheConf, ostream, blockSize, compress, + comparator); } @Override public Writer createWriter(final FSDataOutputStream ostream, final int blockSize, final Compression.Algorithm compress, final KeyComparator c) throws IOException { - return new HFileWriterV2(conf, ostream, blockSize, compress, c); + return new HFileWriterV2(conf, cacheConf, ostream, blockSize, compress, + c); } } /** Constructor that uses all defaults for compression and block size. */ - public HFileWriterV2(Configuration conf, FileSystem fs, Path path) + public HFileWriterV2(Configuration conf, CacheConfig cacheConf, + FileSystem fs, Path path) throws IOException { - this(conf, fs, path, HFile.DEFAULT_BLOCKSIZE, + this(conf, cacheConf, fs, path, HFile.DEFAULT_BLOCKSIZE, HFile.DEFAULT_COMPRESSION_ALGORITHM, null); } @@ -115,38 +120,38 @@ * Constructor that takes a path, creates and closes the output stream. Takes * compression algorithm name as string. */ - public HFileWriterV2(Configuration conf, FileSystem fs, Path path, - int blockSize, String compressAlgoName, + public HFileWriterV2(Configuration conf, CacheConfig cacheConf, FileSystem fs, + Path path, int blockSize, String compressAlgoName, final KeyComparator comparator) throws IOException { - this(conf, fs, path, blockSize, + this(conf, cacheConf, fs, path, blockSize, compressionByName(compressAlgoName), comparator); } /** Constructor that takes a path, creates and closes the output stream. */ - public HFileWriterV2(Configuration conf, FileSystem fs, Path path, - int blockSize, Compression.Algorithm compressAlgo, + public HFileWriterV2(Configuration conf, CacheConfig cacheConf, FileSystem fs, + Path path, int blockSize, Compression.Algorithm compressAlgo, final KeyComparator comparator) throws IOException { - super(conf, createOutputStream(conf, fs, path), path, + super(cacheConf, createOutputStream(conf, fs, path), path, blockSize, compressAlgo, comparator); finishInit(conf); } /** Constructor that takes a stream. */ - public HFileWriterV2(final Configuration conf, + public HFileWriterV2(final Configuration conf, final CacheConfig cacheConf, final FSDataOutputStream outputStream, final int blockSize, final String compressAlgoName, final KeyComparator comparator) throws IOException { - this(conf, outputStream, blockSize, + this(conf, cacheConf, outputStream, blockSize, Compression.getCompressionAlgorithmByName(compressAlgoName), comparator); } /** Constructor that takes a stream. */ - public HFileWriterV2(final Configuration conf, + public HFileWriterV2(final Configuration conf, final CacheConfig cacheConf, final FSDataOutputStream outputStream, final int blockSize, final Compression.Algorithm compress, final KeyComparator comparator) throws IOException { - super(conf, outputStream, null, blockSize, compress, comparator); + super(cacheConf, outputStream, null, blockSize, compress, comparator); finishInit(conf); } @@ -159,9 +164,10 @@ fsBlockWriter = new HFileBlock.Writer(compressAlgo); // Data block index writer + boolean cacheIndexesOnWrite = cacheConf.shouldCacheIndexesOnWrite(); dataBlockIndexWriter = new HFileBlockIndex.BlockIndexWriter(fsBlockWriter, - cacheIndexBlocksOnWrite ? blockCache : null, - cacheIndexBlocksOnWrite ? name : null); + cacheIndexesOnWrite ? cacheConf.getBlockCache(): null, + cacheIndexesOnWrite ? name : null); dataBlockIndexWriter.setMaxChunkSize( HFileBlockIndex.getMaxChunkSize(conf)); inlineBlockWriters.add(dataBlockIndexWriter); @@ -208,8 +214,9 @@ HFile.writeTimeNano.addAndGet(System.nanoTime() - startTimeNs); HFile.writeOps.incrementAndGet(); - if (cacheDataBlocksOnWrite) { - blockCache.cacheBlock(HFile.getBlockCacheKey(name, lastDataBlockOffset), + if (cacheConf.shouldCacheDataOnWrite()) { + cacheConf.getBlockCache().cacheBlock( + HFile.getBlockCacheKey(name, lastDataBlockOffset), fsBlockWriter.getBlockForCaching()); } } @@ -228,7 +235,8 @@ if (cacheThisBlock) { // Cache this block on write. - blockCache.cacheBlock(HFile.getBlockCacheKey(name, offset), + cacheConf.getBlockCache().cacheBlock( + HFile.getBlockCacheKey(name, offset), fsBlockWriter.getBlockForCaching()); } } @@ -242,7 +250,8 @@ */ private void newBlock() throws IOException { // This is where the next block begins. - fsBlockWriter.startWriting(BlockType.DATA, cacheDataBlocksOnWrite); + fsBlockWriter.startWriting(BlockType.DATA, + cacheConf.shouldCacheDataOnWrite()); firstKeyInBlock = null; } @@ -370,7 +379,7 @@ long offset = outputStream.getPos(); // write the metadata content DataOutputStream dos = fsBlockWriter.startWriting(BlockType.META, - cacheDataBlocksOnWrite); + cacheConf.shouldCacheDataOnWrite()); metaData.get(i).write(dos); fsBlockWriter.writeHeaderAndData(outputStream); @@ -424,8 +433,6 @@ @Override public void addInlineBlockWriter(InlineBlockWriter ibw) { inlineBlockWriters.add(ibw); - if (blockCache == null && ibw.cacheOnWrite()) - initBlockCache(); } @Override Index: src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java (working copy) @@ -77,25 +77,13 @@ /** Size of this file. */ protected final long fileSize; - /** Block cache to use. */ - protected final BlockCache blockCache; + /** Block cache configuration. */ + protected final CacheConfig cacheConf; protected AtomicLong cacheHits = new AtomicLong(); protected AtomicLong blockLoads = new AtomicLong(); protected AtomicLong metaLoads = new AtomicLong(); - /** - * Whether file is from in-memory store (comes from column family - * configuration). - */ - protected boolean inMemory = false; - - /** - * Whether blocks of file should be evicted from the block cache when the - * file is being closed - */ - protected final boolean evictOnClose; - /** Path of file */ protected final Path path; @@ -110,16 +98,13 @@ protected AbstractHFileReader(Path path, FixedFileTrailer trailer, final FSDataInputStream fsdis, final long fileSize, final boolean closeIStream, - final BlockCache blockCache, final boolean inMemory, - final boolean evictOnClose) { + final CacheConfig cacheConf) { this.trailer = trailer; this.compressAlgo = trailer.getCompressionCodec(); - this.blockCache = blockCache; + this.cacheConf = cacheConf; this.fileSize = fileSize; this.istream = fsdis; this.closeIStream = closeIStream; - this.inMemory = inMemory; - this.evictOnClose = evictOnClose; this.path = path; this.name = path.getName(); cfStatsPrefix = "cf." + parseCfNameFromPath(path.toString()); @@ -167,7 +152,7 @@ return "reader=" + path.toString() + (!isFileInfoLoaded()? "": ", compression=" + compressAlgo.getName() + - ", inMemory=" + inMemory + + ", cacheConf=" + cacheConf + ", firstKey=" + toStringFirstKey() + ", lastKey=" + toStringLastKey()) + ", avgKeyLen=" + avgKeyLen + Index: src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java (revision 1182021) +++ src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java (working copy) @@ -27,7 +27,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.util.Bytes; @@ -59,10 +59,10 @@ * @param r * @throws IOException */ - public HalfStoreFileReader(final FileSystem fs, final Path p, final BlockCache c, - final Reference r) + public HalfStoreFileReader(final FileSystem fs, final Path p, + final CacheConfig cacheConf, final Reference r) throws IOException { - super(fs, p, c, false, false); + super(fs, p, cacheConf); // This is not actual midkey for this half-file; its just border // around which we split top and bottom. Have to look in files to find // actual last and first keys for bottom and top halves. Half-files don't