Index: hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (date 1550203129000) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (date 1578708338199) @@ -1143,10 +1143,12 @@ // Start log cleaner thread int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 1000); + Map param = new HashMap(); + param.put(MASTER, this); this.logCleaner = new LogCleaner(cleanerInterval, this, conf, getMasterFileSystem().getFileSystem(), - getMasterFileSystem().getOldLogDir()); + getMasterFileSystem().getOldLogDir(), param); getChoreService().scheduleChore(logCleaner); //start the hfile archive cleaner thread Index: hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java (date 1550203129000) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java (date 1578708338206) @@ -27,9 +27,11 @@ import java.io.IOException; import java.lang.reflect.Field; import java.net.URLEncoder; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import com.google.common.collect.Lists; @@ -40,8 +42,10 @@ import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.CoordinatedStateManager; +import org.apache.hadoop.hbase.CoordinatedStateManagerFactory; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerName; @@ -102,6 +106,11 @@ HConstants.HREGION_OLDLOGDIR_NAME); String fakeMachineName = URLEncoder.encode(server.getServerName().toString(), "UTF8"); + CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager( + TEST_UTIL.getConfiguration()); + HMaster master = new HMaster(TEST_UTIL.getConfiguration(), cp); + Map param = new HashMap(); + param.put("master", master); final FileSystem fs = FileSystem.get(conf); @@ -144,7 +153,7 @@ assertEquals(34, fs.listStatus(oldLogDir).length); - LogCleaner cleaner = new LogCleaner(1000, server, conf, fs, oldLogDir); + LogCleaner cleaner = new LogCleaner(1000, server, conf, fs, oldLogDir, param); cleaner.chore(); Index: hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotLogCleanerBug.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotLogCleanerBug.java (date 1578655305000) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotLogCleanerBug.java (date 1578655305000) @@ -0,0 +1,93 @@ +package org.apache.hadoop.hbase.master.cleaner; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CoordinatedStateManager; +import org.apache.hadoop.hbase.CoordinatedStateManagerFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.snapshot.SnapshotLogCleaner; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertFalse; + +@Category(SmallTests.class) +public class TestSnapshotLogCleanerBug { + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniZKCluster(); + } + + @AfterClass + public static void cleanup() throws IOException { + Configuration conf = TEST_UTIL.getConfiguration(); + Path rootDir = FSUtils.getRootDir(conf); + FileSystem fs = FileSystem.get(conf); + // cleanup + fs.delete(rootDir, true); + System.out.println("Finish"); + } + + @Test + public void testSnapshotLogCleaning() throws IOException, KeeperException, InterruptedException { + Server server = new TestLogsCleaner.DummyServer(); + Configuration conf = TEST_UTIL.getConfiguration(); + final FileSystem fs = FileSystem.get(conf); + FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir()); + Path rootDir = FSUtils.getRootDir(conf); + final Path oldLogDir = new Path(TEST_UTIL.getDataTestDir(), + HConstants.HREGION_OLDLOGDIR_NAME); + CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager( + TEST_UTIL.getConfiguration()); + HMaster master = new HMaster(TEST_UTIL.getConfiguration(), cp); + Map param = new HashMap(); + param.put("master", master); + + LogCleaner logCleaner = new LogCleaner(1000, server, conf, fs, oldLogDir, param); + + SnapshotLogCleaner cleaner = new SnapshotLogCleaner(); + cleaner.setConf(conf); + cleaner.init(logCleaner.params); + + // write an hfile to the snapshot directory + String snapshotName = "snapshot"; + byte[] snapshot = Bytes.toBytes(snapshotName); + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); + Path snapshotLogDir = new Path(snapshotDir, HConstants.HREGION_LOGDIR_NAME); + String timestamp = "1339643343027"; + String hostFromMaster = "localhost%2C59648%2C1339643336601"; + + Path hostSnapshotLogDir = new Path(snapshotLogDir, hostFromMaster); + String snapshotlogfile = hostFromMaster + "." + timestamp + ".hbase"; + + // add the reference to log in the snapshot + fs.create(new Path(hostSnapshotLogDir, snapshotlogfile)); + + // now check to see if that log file would get deleted. + Path oldlogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME); + Path logFile = new Path(oldlogDir, snapshotlogfile); + fs.create(logFile); + + // make sure that the file isn't deletable + assertFalse(cleaner.isFileDeletable(fs.getFileStatus(logFile))); + } + +} \ No newline at end of file Index: hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java (date 1550203129000) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java (date 1578708338187) @@ -28,6 +28,8 @@ import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.wal.DefaultWALProvider; +import java.util.Map; + /** * This Chore, every time it runs, will attempt to delete the WALs in the old logs folder. The WAL * is only deleted if none of the cleaner delegates says otherwise. @@ -45,8 +47,8 @@ * @param oldLogDir the path to the archived logs */ public LogCleaner(final int p, final Stoppable s, Configuration conf, FileSystem fs, - Path oldLogDir) { - super("LogsCleaner", p, s, conf, fs, oldLogDir, HBASE_MASTER_LOGCLEANER_PLUGINS); + Path oldLogDir, Map params) { + super("LogsCleaner", p, s, conf, fs, oldLogDir, HBASE_MASTER_LOGCLEANER_PLUGINS, params); } @Override