diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index 412f9b3..56458a0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -49,8 +49,10 @@ import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.backup.HFileArchiver; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.Reference; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSHDFSUtils; +import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; @@ -577,9 +579,8 @@ public class HRegionFileSystem { * @throws IOException */ Path splitStoreFile(final HRegionInfo hri, final String familyName, final StoreFile f, - final byte[] splitRow, final boolean top, RegionSplitPolicy splitPolicy) + final byte[] splitRow, final boolean top, RegionSplitPolicy splitPolicy,CacheConfig cacheConf) throws IOException { - if (splitPolicy == null || !splitPolicy.skipStoreFileRangeCheck(familyName)) { // Check whether the split row lies in the range of the store file // If it is outside the range, return directly. @@ -608,7 +609,7 @@ public class HRegionFileSystem { } } } finally { - f.closeReader(true); + f.closeReader(cacheConf != null? cacheConf.shouldEvictOnClose(): true); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 133a5d9..184d44f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -570,9 +570,11 @@ public class HStore implements Store { } if (ioe != null) { // close StoreFile readers + boolean evictOnClose = + cacheConf != null? cacheConf.shouldEvictOnClose(): true; for (StoreFile file : results) { try { - if (file != null) file.closeReader(true); + if (file != null) file.closeReader(evictOnClose); } catch (IOException e) { LOG.warn(e.getMessage()); } @@ -1244,10 +1246,12 @@ public class HStore implements Store { if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) { LOG.warn("hbase.hstore.compaction.complete is set to false"); sfs = new ArrayList(newFiles.size()); + final boolean evictOnClose = + cacheConf != null? cacheConf.shouldEvictOnClose(): true; for (Path newFile : newFiles) { // Create storefile around what we wrote with a reader on it. StoreFile sf = createStoreFileAndReader(newFile); - sf.closeReader(true); + sf.closeReader(evictOnClose); sfs.add(sf); } return sfs; @@ -1797,8 +1801,10 @@ public class HStore implements Store { // let the archive util decide if we should archive or delete the files LOG.debug("Removing store files after compaction..."); + boolean evictOnClose = + cacheConf != null? cacheConf.shouldEvictOnClose(): true; for (StoreFile compactedFile : compactedFiles) { - compactedFile.closeReader(true); + compactedFile.closeReader(evictOnClose); } if (removeFiles) { this.fs.removeStoreFiles(this.getColumnFamilyName(), compactedFiles); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java index 70d040e..346c10b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java @@ -39,15 +39,18 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CancelableProgressable; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HasThread; import org.apache.hadoop.hbase.util.Pair; @@ -671,10 +674,12 @@ public class SplitTransactionImpl implements SplitTransaction { (ThreadPoolExecutor) Executors.newFixedThreadPool(maxThreads, factory); List>> futures = new ArrayList>> (nbFiles); + FSTableDescriptors tableDescs = new FSTableDescriptors(parent.getBaseConf()); + HTableDescriptor desc = tableDescs.get(parent.getRegionInfo().getTable()); // Split each store file. for (Map.Entry> entry: hstoreFilesToSplit.entrySet()) { for (StoreFile sf: entry.getValue()) { - StoreFileSplitter sfs = new StoreFileSplitter(entry.getKey(), sf); + StoreFileSplitter sfs = new StoreFileSplitter(entry.getKey(), sf, desc); futures.add(threadPool.submit(sfs)); } } @@ -720,7 +725,8 @@ public class SplitTransactionImpl implements SplitTransaction { return new Pair(created_a, created_b); } - private Pair splitStoreFile(final byte[] family, final StoreFile sf) + private Pair splitStoreFile(final byte[] family, final StoreFile sf, + final CacheConfig cacheConf) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("Splitting started for store file: " + sf.getPath() + " for region: " + @@ -730,10 +736,10 @@ public class SplitTransactionImpl implements SplitTransaction { String familyName = Bytes.toString(family); Path path_a = fs.splitStoreFile(this.hri_a, familyName, sf, this.splitrow, false, - this.parent.getSplitPolicy()); + this.parent.getSplitPolicy(), cacheConf); Path path_b = fs.splitStoreFile(this.hri_b, familyName, sf, this.splitrow, true, - this.parent.getSplitPolicy()); + this.parent.getSplitPolicy(), cacheConf); if (LOG.isDebugEnabled()) { LOG.debug("Splitting complete for store file: " + sf.getPath() + " for region: " + this.parent); @@ -748,19 +754,23 @@ public class SplitTransactionImpl implements SplitTransaction { private class StoreFileSplitter implements Callable> { private final byte[] family; private final StoreFile sf; + private final HTableDescriptor desc; /** * Constructor that takes what it needs to split * @param family Family that contains the store file * @param sf which file + * @param desc table descriptor */ - public StoreFileSplitter(final byte[] family, final StoreFile sf) { + public StoreFileSplitter(final byte[] family, final StoreFile sf, HTableDescriptor desc) { this.sf = sf; this.family = family; + this.desc = desc; } public Pair call() throws IOException { - return splitStoreFile(family, sf); + CacheConfig cacheConf = new CacheConfig(parent.getBaseConf(), desc.getFamily(family)); + return splitStoreFile(family, sf, cacheConf); } } @@ -778,6 +788,7 @@ public class SplitTransactionImpl implements SplitTransaction { throws IOException { this.server = server; this.rsServices = services; + // Coprocessor callback if (this.parent.getCoprocessorHost() != null) { if (user == null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java index 88aa151..36abb83 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java @@ -515,7 +515,9 @@ public class StoreFile { this.reader = open(canUseDropBehind); } catch (IOException e) { try { - this.closeReader(true); + boolean evictOnClose = + cacheConf != null? cacheConf.shouldEvictOnClose(): true; + this.closeReader(evictOnClose); } catch (IOException ee) { } throw e; @@ -550,7 +552,9 @@ public class StoreFile { * @throws IOException */ public void deleteReader() throws IOException { - closeReader(true); + boolean evictOnClose = + cacheConf != null? cacheConf.shouldEvictOnClose(): true; + closeReader(evictOnClose); this.fs.delete(getPath(), true); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java index 4b12cff..5c4be4c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java @@ -1065,11 +1065,13 @@ public class TestSplitTransactionOnCluster { assertFalse(region.hasReferences()); Path referencePath = region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "f", - storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy()); + storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy(), + null); assertNull(referencePath); referencePath = region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "i_f", - storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy()); + storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy(), + null); assertNotNull(referencePath); } finally { TESTING_UTIL.deleteTable(tableName); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java index 85e4439..c7e36c7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java @@ -965,7 +965,7 @@ public class TestStoreFile extends HBaseTestCase { final String family, final StoreFile sf, final byte[] splitKey, boolean isTopRef) throws IOException { FileSystem fs = regionFs.getFileSystem(); - Path path = regionFs.splitStoreFile(hri, family, sf, splitKey, isTopRef, null); + Path path = regionFs.splitStoreFile(hri, family, sf, splitKey, isTopRef, null, null); if (null == path) { return null; }