From f6c984d212cae5f0004b7b8d4c519baad075bf3a Mon Sep 17 00:00:00 2001 From: chenheng Date: Mon, 22 Aug 2016 16:45:20 +0800 Subject: [PATCH] HBASE-16464 archive folder grows bigger and bigger due to corrupt snapshot under tmp dir --- .../hbase/master/snapshot/SnapshotFileCache.java | 12 +++++++++++- .../hbase/snapshot/SnapshotDescriptionUtils.java | 1 + .../master/snapshot/TestSnapshotHFileCleaner.java | 20 ++++++++++++++++++++ .../hadoop/hbase/snapshot/SnapshotTestingUtils.java | 12 ++++++++++++ 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java index dfd3cb5..6ccfcc5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java @@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.util.FSUtils; @@ -300,7 +301,16 @@ public class SnapshotFileCache implements Stoppable { FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir); if (running != null) { for (FileStatus run : running) { - snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath())); + try { + snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath())); + } catch (CorruptedSnapshotException e) { + if (e.getCause() instanceof FileNotFoundException) { + fs.delete(run.getPath(), true); + LOG.warn("delete the " + run.getPath() + " due to exception:", e.getCause()); + } else { + throw e; + } + } } } return snapshotInProgress; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java index 560a42b..8cc36f5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.snapshot; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.Collections; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java index b4355ba..88ca5a0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java @@ -168,4 +168,24 @@ public class TestSnapshotHFileCleaner { fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true); } } + + + /** + * HBASE-16464 + */ + @Test + public void testMissedTmpSnapshot() throws IOException { + SnapshotTestingUtils.SnapshotMock + snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); + SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2( + SNAPSHOT_NAME_STR, TABLE_NAME_STR); + builder.addRegionV2(); + builder.missOneRegionSnapshotFile(); + + long period = Long.MAX_VALUE; + SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, + "test-snapshot-file-cache-refresh", new SnapshotFiles()); + cache.getSnapshotsInProgress(); + assertFalse(fs.exists(builder.getSnapshotsDir())); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java index 38afc3b..770bb00 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java @@ -583,6 +583,18 @@ public final class SnapshotTestingUtils { corruptFile(p); } + public void missOneRegionSnapshotFile() throws IOException { + FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir); + for (FileStatus fileStatus : manifestFiles) { + String fileName = fileStatus.getPath().getName(); + if (fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOTINFO_FILE) + || fileName.endsWith(".tabledesc") + || fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) { + fs.delete(fileStatus.getPath(), true); + } + } + } + /** * Corrupt data-manifest file * -- 1.9.3 (Apple Git-50)