From 218c5bec33b03658dd343f627bb449f758927f8f Mon Sep 17 00:00:00 2001 From: Artem Ervits Date: Thu, 25 Oct 2018 16:50:49 -0400 Subject: [PATCH] HBASE-21175 Partially initialized SnapshotHFileCleaner leads to NPE during TestHFileArchiving --- .../hadoop/hbase/master/cleaner/HFileCleaner.java | 9 ++++++++ .../hadoop/hbase/backup/TestHFileArchiving.java | 24 ++++++++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java index 47b0228a31..7ad6177764 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java @@ -32,6 +32,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.util.StealJobQueue; import org.apache.yetus.audience.InterfaceAudience; @@ -44,6 +46,7 @@ import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesti */ @InterfaceAudience.Private public class HFileCleaner extends CleanerChore { + private MasterServices master; public static final String MASTER_HFILE_CLEANER_PLUGINS = "hbase.master.hfilecleaner.plugins"; @@ -496,4 +499,10 @@ public class HFileCleaner extends CleanerChore { t.interrupt(); } } + + public void init(Map params) { + if (params != null && params.containsKey(HMaster.MASTER)) { + this.master = (MasterServices) params.get(HMaster.MASTER); + } + } } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java index dfebd38a0a..479c001f2b 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java @@ -24,7 +24,10 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Objects; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -33,12 +36,11 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; -import org.apache.hadoop.hbase.regionserver.CompactingMemStore; import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; -import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.Bytes; @@ -49,6 +51,7 @@ import org.apache.hadoop.hbase.util.StoppableImplementation; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; +import static org.junit.Assert.assertNotNull; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Rule; @@ -209,7 +212,7 @@ public class TestHFileArchiving { PathFilter nonHidden = new PathFilter() { @Override public boolean accept(Path file) { - return dirFilter.accept(file) && !file.getName().toString().startsWith("."); + return dirFilter.accept(file) && !file.getName().startsWith("."); } }; FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden); @@ -374,11 +377,10 @@ public class TestHFileArchiving { Stoppable stoppable = new StoppableImplementation(); // The cleaner should be looping without long pauses to reproduce the race condition. - HFileCleaner cleaner = new HFileCleaner(1, stoppable, conf, fs, archiveDir); - assertFalse("cleaner should not be null", cleaner == null); + HFileCleaner cleaner = getHFileCleaner(stoppable, conf, fs, archiveDir); + assertNotNull("cleaner should not be null", cleaner); try { choreService.scheduleChore(cleaner); - // Keep creating/archiving new files while the cleaner is running in the other thread long startTime = System.currentTimeMillis(); for (long fid = 0; (System.currentTimeMillis() - startTime) < TEST_TIME; ++fid) { @@ -418,6 +420,16 @@ public class TestHFileArchiving { } } + // Avoid passing a null master to CleanerChore, see HBASE-21175 + private HFileCleaner getHFileCleaner(Stoppable stoppable, Configuration conf, + FileSystem fs, Path archiveDir) throws IOException { + Map params = new HashMap<>(); + params.put(HMaster.MASTER, UTIL.getMiniHBaseCluster().getMaster()); + HFileCleaner cleaner = new HFileCleaner(1, stoppable, conf, fs, archiveDir); + cleaner.init(params); + return Objects.requireNonNull(cleaner); + } + private void clearArchiveDirectory() throws IOException { UTIL.getTestFileSystem().delete( new Path(UTIL.getDefaultRootDirPath(), HConstants.HFILE_ARCHIVE_DIRECTORY), true); -- 2.16.2