From 8ac715123caeae0a2ba6d1b655c02732960cce35 Mon Sep 17 00:00:00 2001 From: Vladimir Rodionov Date: Tue, 21 Aug 2018 12:35:00 -0700 Subject: [PATCH] HBASE-21077: MR job launched by hbase incremental backup command failed with FileNotFoundException --- .../apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java | 2 +- .../test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java | 7 +++++++ .../java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java index 81652d8ae2..3eebf4299c 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java @@ -178,7 +178,7 @@ public class IncrementalBackupManager extends BackupManager { Set set = new HashSet<>(); for (int i=0; i < logFromSystemTable.size(); i++) { WALItem item = logFromSystemTable.get(i); - set.add(item.walFile); + set.add((new Path(item.walFile)).getName()); } return set; } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java index 08ecd632c6..bbac2f3037 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java @@ -57,6 +57,8 @@ import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.master.cleaner.LogCleaner; +import org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner; import org.apache.hadoop.hbase.security.HadoopSecurityEnabledUserProviderForTesting; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.security.access.SecureTestUtil; @@ -288,6 +290,11 @@ public class TestBackupBase { BackupManager.decorateMasterConfiguration(conf1); BackupManager.decorateRegionServerConfiguration(conf1); conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); + // Set TTL for old WALs to 5 sec to enforce fast cleaning of an archived + // WAL files + conf1.setLong(TimeToLiveLogCleaner.TTL_CONF_KEY, 1000); + conf1.setLong(LogCleaner.OLD_WALS_CLEANER_THREAD_TIMEOUT_MSEC, 1000); + // Set MultiWAL (with 2 default WAL files per RS) conf1.set(WALFactory.WAL_PROVIDER, provider); TEST_UTIL.startMiniCluster(); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java index b74f42fb83..48e2c5eb8b 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java @@ -157,6 +157,8 @@ public class TestIncrementalBackup extends TestBackupBase { int NB_ROWS_FAM2 = 7; HTable t3 = insertIntoTable(conn, table1, fam2Name, 2, NB_ROWS_FAM2); t3.close(); + // Wait for 5 sec to make sure that old WALs were deleted + Thread.sleep(5000); // #3 - incremental backup for multiple tables request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); -- 2.14.3 (Apple Git-98)