From 7fc802056fc7847729c0c31b8eb96f2177331d32 Mon Sep 17 00:00:00 2001 From: Pankaj Kumar Date: Fri, 14 Oct 2016 20:02:44 +0800 Subject: [PATCH] HBASE-16716, OfflineMetaRepair leaves empty directory inside /hbase/WALs which remains forever. --- .../org/apache/hadoop/hbase/util/HBaseFsck.java | 21 +++++++++++++++++ .../util/hbck/TestOfflineMetaRebuildBase.java | 26 ++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index 51f0218..4d9d2db 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -123,6 +123,7 @@ import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.wal.FSHLog; import org.apache.hadoop.hbase.security.AccessDeniedException; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator; @@ -1437,12 +1438,32 @@ public class HBaseFsck extends Configured implements Closeable { } meta.batchMutate(puts.toArray(new Put[puts.size()])); HRegion.closeHRegion(meta); + // clean up the temporary hbck meta recovery WAL directory + removeHBCKMetaRecoveryWALDir(meta); LOG.info("Success! hbase:meta table rebuilt."); LOG.info("Old hbase:meta is moved into " + backupDir); return true; } /** + * Removes the empty Meta recovery WAL directory. + * @param meta Meta region + */ + private void removeHBCKMetaRecoveryWALDir(HRegion meta) throws IOException { + // TODO Since HBASE-11983 not available in this branch, so we don't know the walFactoryId. + // Retrieving WAL directory + Path walLogDir = ((FSHLog) meta.getWAL()).getCurrentFileName().getParent(); + FileSystem fs = FSUtils.getCurrentFileSystem(getConf()); + FileStatus[] walFiles = FSUtils.listStatus(fs, walLogDir, null); + if (walFiles == null || walFiles.length == 0) { + LOG.info("HBCK meta recovery WAL directory is empty, removing it now."); + if (!FSUtils.deleteDirectory(fs, walLogDir)) { + LOG.warn("Couldn't clear the HBCK Meta recovery WAL directory " + walLogDir); + } + } + } + + /** * Log an appropriate message about whether or not overlapping merges are computed in parallel. */ private void logParallelMerge() { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java index f4a035f..f286c22 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java @@ -20,15 +20,22 @@ package org.apache.hadoop.hbase.util.hbck; import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors; import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import java.io.IOException; import java.util.Arrays; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HBaseFsck; import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; import org.junit.Test; @@ -62,6 +69,7 @@ public class TestOfflineMetaRebuildBase extends OfflineMetaRebuildTestCore { // rebuild meta table from scratch HBaseFsck fsck = new HBaseFsck(conf); assertTrue(fsck.rebuildMeta(false)); + assertTrue("HBCK meta recovery WAL directory exist.", validateHBCKMetaRecoveryWALDir()); // bring up the minicluster TEST_UTIL.startMiniZKCluster(); @@ -84,4 +92,22 @@ public class TestOfflineMetaRebuildBase extends OfflineMetaRebuildTestCore { LOG.info("Table " + table + " has " + tableRowCount(conf, table) + " entries."); assertEquals(16, tableRowCount(conf, table)); } + + /** + * Validate whether Meta recovery empty WAL directory is removed. + * @return True if directory is removed otherwise false. + */ + private boolean validateHBCKMetaRecoveryWALDir() throws IOException { + Path rootdir = FSUtils.getRootDir(TEST_UTIL.getConfiguration()); + Path walLogDir = new Path(rootdir, HConstants.HREGION_LOGDIR_NAME); + FileSystem fs = TEST_UTIL.getTestFileSystem(); + FileStatus[] walFiles = FSUtils.listStatus(fs, walLogDir, null); + assertNotNull(walFiles); + for (FileStatus fsStat : walFiles) { + if (fsStat.isDirectory() && fsStat.getPath().getName().startsWith("hregion-")) { + return false; + } + } + return true; + } } -- 2.7.2.windows.1