From 8bb6c7f4256183cf6581cb7269a57e417f5684a8 Mon Sep 17 00:00:00 2001 From: Guanghao Zhang Date: Thu, 4 Apr 2019 16:54:54 +0800 Subject: [PATCH] HBASE-22163 May send a warmup rpc for a splited parent region --- .../example/TestRefreshHFilesEndpoint.java | 6 ++-- .../hadoop/hbase/regionserver/CompactionTool.java | 2 +- .../hadoop/hbase/regionserver/HMobStore.java | 4 +-- .../apache/hadoop/hbase/regionserver/HRegion.java | 13 ++++--- .../apache/hadoop/hbase/regionserver/HStore.java | 40 ++++++++++++---------- .../org/apache/hadoop/hbase/TestIOFencing.java | 10 +++--- .../client/TestFromClientSideScanExcpetion.java | 11 +++--- .../regionserver/TestCacheOnWriteInSchema.java | 2 +- .../hbase/regionserver/TestCompactingMemStore.java | 2 +- .../hbase/regionserver/TestCompactionPolicy.java | 2 +- .../hadoop/hbase/regionserver/TestHMobStore.java | 2 +- .../hadoop/hbase/regionserver/TestHRegion.java | 18 +++++----- .../hadoop/hbase/regionserver/TestHStore.java | 9 ++--- 13 files changed, 67 insertions(+), 54 deletions(-) diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java index 1a7c990..3f9c23b 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java @@ -109,9 +109,9 @@ public class TestRefreshHFilesEndpoint extends TestRefreshHFilesBase { } public static class HStoreWithFaultyRefreshHFilesAPI extends HStore { - public HStoreWithFaultyRefreshHFilesAPI(final HRegion region, final ColumnFamilyDescriptor family, - final Configuration confParam) throws IOException { - super(region, family, confParam); + public HStoreWithFaultyRefreshHFilesAPI(final HRegion region, + final ColumnFamilyDescriptor family, final Configuration confParam) throws IOException { + super(region, family, confParam, false); } @Override diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java index 18a5987..da06fff 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java @@ -191,7 +191,7 @@ public class CompactionTool extends Configured implements Tool { } }; HRegion region = new HRegion(regionFs, null, conf, htd, null); - return new HStore(region, htd.getColumnFamily(Bytes.toBytes(familyName)), conf); + return new HStore(region, htd.getColumnFamily(Bytes.toBytes(familyName)), conf, false); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java index b657183..632642f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java @@ -104,8 +104,8 @@ public class HMobStore extends HStore { private final byte[] refCellTags; public HMobStore(final HRegion region, final ColumnFamilyDescriptor family, - final Configuration confParam) throws IOException { - super(region, family, confParam); + final Configuration confParam, boolean warmup) throws IOException { + super(region, family, confParam, warmup); this.family = family; this.mobFileCache = region.getMobFileCache(); this.homePath = MobUtils.getMobHome(conf); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index bfd026a..3bad79d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -1052,6 +1052,11 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi */ private long initializeStores(CancelableProgressable reporter, MonitoredTask status) throws IOException { + return initializeStores(reporter, status, false); + } + + private long initializeStores(CancelableProgressable reporter, MonitoredTask status, + boolean warmup) throws IOException { // Load in all the HStores. long maxSeqId = -1; // initialized to -1 so that we pick up MemstoreTS from column families @@ -1069,7 +1074,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi completionService.submit(new Callable() { @Override public HStore call() throws IOException { - return instantiateHStore(family); + return instantiateHStore(family, warmup); } }); } @@ -5814,7 +5819,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi return true; } - protected HStore instantiateHStore(final ColumnFamilyDescriptor family) throws IOException { + protected HStore instantiateHStore(final ColumnFamilyDescriptor family, boolean warmup) throws IOException { if (family.isMobEnabled()) { if (HFile.getFormatVersion(this.conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) { throw new IOException("A minimum HFile version of " @@ -5822,9 +5827,9 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi + " is required for MOB feature. Consider setting " + HFile.FORMAT_VERSION_KEY + " accordingly."); } - return new HMobStore(this, family, this.conf); + return new HMobStore(this, family, this.conf, warmup); } - return new HStore(this, family, this.conf); + return new HStore(this, family, this.conf, warmup); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 79022a1..986c62a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -241,7 +241,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat * @throws IOException */ protected HStore(final HRegion region, final ColumnFamilyDescriptor family, - final Configuration confParam) throws IOException { + final Configuration confParam, boolean warmup) throws IOException { this.fs = region.getRegionFileSystem(); @@ -303,7 +303,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat } this.storeEngine = createStoreEngine(this, this.conf, this.comparator); - List hStoreFiles = loadStoreFiles(); + List hStoreFiles = loadStoreFiles(warmup); // Move the storeSize calculation out of loadStoreFiles() method, because the secondary read // replica's refreshStoreFiles() will also use loadStoreFiles() to refresh its store files and // update the storeSize in the completeCompaction(..) finally (just like compaction) , so @@ -555,12 +555,13 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat * from the given directory. * @throws IOException */ - private List loadStoreFiles() throws IOException { + private List loadStoreFiles(boolean warmup) throws IOException { Collection files = fs.getStoreFiles(getColumnFamilyName()); - return openStoreFiles(files); + return openStoreFiles(files, warmup); } - private List openStoreFiles(Collection files) throws IOException { + private List openStoreFiles(Collection files, boolean warmup) + throws IOException { if (CollectionUtils.isEmpty(files)) { return Collections.emptyList(); } @@ -614,19 +615,22 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat throw ioe; } - // Remove the compacted files from result - List filesToRemove = new ArrayList<>(compactedStoreFiles.size()); - for (HStoreFile storeFile : results) { - if (compactedStoreFiles.contains(storeFile.getPath().getName())) { - LOG.warn("Clearing the compacted storefile {} from this store", storeFile); - storeFile.getReader().close(true); - filesToRemove.add(storeFile); + // Only archived the compacted store files when real open + if (!warmup) { + // Remove the compacted files from result + List filesToRemove = new ArrayList<>(compactedStoreFiles.size()); + for (HStoreFile storeFile : results) { + if (compactedStoreFiles.contains(storeFile.getPath().getName())) { + LOG.warn("Clearing the compacted storefile {} from this store", storeFile); + storeFile.getReader().close(true); + filesToRemove.add(storeFile); + } + } + results.removeAll(filesToRemove); + if (!filesToRemove.isEmpty() && this.isPrimaryReplicaStore()) { + LOG.debug("Moving the files {} to archive", filesToRemove); + this.fs.removeStoreFiles(this.getColumnFamilyDescriptor().getNameAsString(), filesToRemove); } - } - results.removeAll(filesToRemove); - if (!filesToRemove.isEmpty() && this.isPrimaryReplicaStore()) { - LOG.debug("Moving the files {} to archive", filesToRemove); - this.fs.removeStoreFiles(this.getColumnFamilyDescriptor().getNameAsString(), filesToRemove); } return results; @@ -694,7 +698,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat } // try to open the files - List openedFiles = openStoreFiles(toBeAddedFiles); + List openedFiles = openStoreFiles(toBeAddedFiles, false); // propogate the file changes to the underlying store file manager replaceStoreFiles(toBeRemovedStoreFiles, openedFiles); //won't throw an exception diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java index 2df484a..063d023 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java @@ -194,17 +194,19 @@ public class TestIOFencing { TableDescriptor htd, RegionServerServices rsServices) { super(tableDir, log, fs, confParam, info, htd, rsServices); } + @Override - protected HStore instantiateHStore(final ColumnFamilyDescriptor family) throws IOException { - return new BlockCompactionsInCompletionHStore(this, family, this.conf); + protected HStore instantiateHStore(final ColumnFamilyDescriptor family, boolean warmup) + throws IOException { + return new BlockCompactionsInCompletionHStore(this, family, this.conf, warmup); } } public static class BlockCompactionsInCompletionHStore extends HStore { CompactionBlockerRegion r; protected BlockCompactionsInCompletionHStore(HRegion region, ColumnFamilyDescriptor family, - Configuration confParam) throws IOException { - super(region, family, confParam); + Configuration confParam, boolean warmup) throws IOException { + super(region, family, confParam, warmup); r = (CompactionBlockerRegion) region; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSideScanExcpetion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSideScanExcpetion.java index 34481ba..3fb482d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSideScanExcpetion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSideScanExcpetion.java @@ -116,16 +116,17 @@ public class TestFromClientSideScanExcpetion { } @Override - protected HStore instantiateHStore(ColumnFamilyDescriptor family) throws IOException { - return new MyHStore(this, family, conf); + protected HStore instantiateHStore(ColumnFamilyDescriptor family, boolean warmup) + throws IOException { + return new MyHStore(this, family, conf, warmup); } } public static final class MyHStore extends HStore { - public MyHStore(HRegion region, ColumnFamilyDescriptor family, Configuration confParam) - throws IOException { - super(region, family, confParam); + public MyHStore(HRegion region, ColumnFamilyDescriptor family, Configuration confParam, + boolean warmup) throws IOException { + super(region, family, confParam, warmup); } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java index 1248b21..5bb103c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java @@ -182,7 +182,7 @@ public class TestCacheOnWriteInSchema { region = TEST_UTIL.createLocalHRegion(info, htd, walFactory.getWAL(info)); region.setBlockCache(BlockCacheFactory.createBlockCache(conf)); - store = new HStore(region, hcd, conf); + store = new HStore(region, hcd, conf, false); } @After diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java index 501d5cd..e5a2a8a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java @@ -118,7 +118,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { this.regionServicesForStores = Mockito.spy(region.getRegionServicesForStores()); ThreadPoolExecutor pool = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); Mockito.when(regionServicesForStores.getInMemoryCompactionPool()).thenReturn(pool); - this.store = new HStore(region, hcd, conf); + this.store = new HStore(region, hcd, conf, false); long globalMemStoreLimit = (long) (ManagementFactory.getMemoryMXBean().getHeapMemoryUsage() .getMax() * MemorySizeUtil.getGlobalMemStoreHeapPercent(conf, false)); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java index 939f35c..90a9596 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java @@ -107,7 +107,7 @@ public class TestCompactionPolicy { Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName()); region = new HRegion(tableDir, hlog, fs, conf, info, htd, null); - store = new HStore(region, hcd, conf); + store = new HStore(region, hcd, conf, false); TEST_FILE = region.getRegionFileSystem().createTempName(); fs.createNewFile(TEST_FILE); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java index 2001602..bf1f18e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java @@ -166,7 +166,7 @@ public class TestHMobStore { final WALFactory wals = new WALFactory(walConf, methodName); region = new HRegion(tableDir, wals.getWAL(info), fs, conf, info, td, null); region.setMobFileCache(new MobFileCache(conf)); - store = new HMobStore(region, cfd, conf); + store = new HMobStore(region, cfd, conf, false); if (testStore) { init(conf, cfd); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 106f271..6af4bb1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -6306,17 +6306,17 @@ public class TestHRegion { * @return If Mob is enabled, return HMobStore, otherwise return HStoreForTesting. */ @Override - protected HStore instantiateHStore(final ColumnFamilyDescriptor family) throws IOException { + protected HStore instantiateHStore(final ColumnFamilyDescriptor family, boolean warmup) + throws IOException { if (family.isMobEnabled()) { if (HFile.getFormatVersion(this.conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) { - throw new IOException("A minimum HFile version of " - + HFile.MIN_FORMAT_VERSION_WITH_TAGS - + " is required for MOB feature. Consider setting " + HFile.FORMAT_VERSION_KEY - + " accordingly."); + throw new IOException("A minimum HFile version of " + HFile.MIN_FORMAT_VERSION_WITH_TAGS + + " is required for MOB feature. Consider setting " + HFile.FORMAT_VERSION_KEY + + " accordingly."); } - return new HMobStore(this, family, this.conf); + return new HMobStore(this, family, this.conf, warmup); } - return new HStoreForTesting(this, family, this.conf); + return new HStoreForTesting(this, family, this.conf, warmup); } } @@ -6332,8 +6332,8 @@ public class TestHRegion { protected HStoreForTesting(final HRegion region, final ColumnFamilyDescriptor family, - final Configuration confParam) throws IOException { - super(region, family, confParam); + final Configuration confParam, boolean warmup) throws IOException { + super(region, family, confParam, warmup); } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java index 786334e..e1e74f4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java @@ -230,7 +230,7 @@ public class TestHStore { ColumnFamilyDescriptor hcd, MyStoreHook hook, boolean switchToPread) throws IOException { initHRegion(methodName, conf, builder, hcd, hook, switchToPread); if (hook == null) { - store = new HStore(region, hcd, conf); + store = new HStore(region, hcd, conf, false); } else { store = new MyStore(region, hcd, conf, hook, switchToPread); } @@ -495,7 +495,8 @@ public class TestHStore { w.close(); this.store.close(); // Reopen it... should pick up two files - this.store = new HStore(this.store.getHRegion(), this.store.getColumnFamilyDescriptor(), c); + this.store = + new HStore(this.store.getHRegion(), this.store.getColumnFamilyDescriptor(), c, false); assertEquals(2, this.store.getStorefilesCount()); result = HBaseTestingUtility.getFromStoreFile(store, @@ -1525,7 +1526,7 @@ public class TestHStore { ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.of(family); initHRegion(name.getMethodName(), conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)), hcd, null, false); - HStore store = new HStore(region, hcd, conf) { + HStore store = new HStore(region, hcd, conf, false) { @Override protected StoreEngine createStoreEngine(HStore store, Configuration conf, @@ -1567,7 +1568,7 @@ public class TestHStore { MyStore(final HRegion region, final ColumnFamilyDescriptor family, final Configuration confParam, MyStoreHook hook, boolean switchToPread) throws IOException { - super(region, family, confParam); + super(region, family, confParam, false); this.hook = hook; } -- 2.7.4