From d0565b64f6989ef2f1c793aa72f638ae2908f904 Mon Sep 17 00:00:00 2001 From: Vladimir Rodionov Date: Tue, 31 Jul 2018 17:42:55 -0700 Subject: [PATCH] HBASE-20729: B&R BackupLogCleaner must ignore ProcV2 WAL files --- .../hadoop/hbase/backup/impl/BackupSystemTable.java | 6 ++++++ .../hbase/backup/impl/IncrementalBackupManager.java | 16 ++++++++++++++++ .../hadoop/hbase/backup/master/BackupLogCleaner.java | 1 - 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java index 1b3694425a..d177384cac 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java @@ -65,6 +65,7 @@ import org.apache.hadoop.hbase.client.SnapshotDescription; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore; import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.util.Bytes; @@ -1116,6 +1117,11 @@ public final class BackupSystemTable implements Closeable { List fileStatuses = new ArrayList<>(); for (FileStatus file : files) { + String fn = file.getPath().getName(); + if (fn.startsWith(WALProcedureStore.LOG_PREFIX)) { + ret.put(file, true); + continue; + } String wal = file.getPath().toString(); Get get = createGetForCheckWALFile(wal); getBuffer.add(get); diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java index a20f9b548f..ea9c5cc0fa 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager; import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.yetus.audience.InterfaceAudience; @@ -102,6 +103,7 @@ public class IncrementalBackupManager extends BackupManager { getLogFilesFromBackupSystem(previousTimestampMins, newTimestamps, getBackupInfo() .getBackupRootDir()); logList = excludeAlreadyBackedUpWALs(logList, logFromSystemTable); + logList = excludeProcV2WALs(logList); backupInfo.setIncrBackupFileList(logList); return newTimestamps; @@ -145,12 +147,26 @@ public class IncrementalBackupManager extends BackupManager { .getBackupRootDir()); logList = excludeAlreadyBackedUpWALs(logList, logFromSystemTable); + logList = excludeProcV2WALs(logList); backupInfo.setIncrBackupFileList(logList); return logList; } + private List excludeProcV2WALs(List logList) { + List ret = new ArrayList(); + for (String sp: logList) { + Path p = new Path(sp); + String fname = p.getName(); + if (fname.startsWith(WALProcedureStore.LOG_PREFIX)) { + continue; + } + ret.add(sp); + } + return ret; + } + private List excludeAlreadyBackedUpWALs(List logList, List logFromSystemTable) { Set walFileNameSet = convertToSet(logFromSystemTable); diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java index 5ce11d19c0..8d9400fd92 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java @@ -97,7 +97,6 @@ public class BackupLogCleaner extends BaseLogCleanerDelegate { LOG.warn("Backup system table is not available: {}", tnfe.getMessage()); return files; } - List list = new ArrayList<>(); Map walFilesDeletableMap = table.areWALFilesDeletable(files); for (Map.Entry entry: walFilesDeletableMap.entrySet()) { -- 2.14.3 (Apple Git-98)