diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java index d44ba4e..f14fc1e 100644 --- hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java +++ hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.BackupProtos; import org.apache.hadoop.hbase.protobuf.generated.BackupProtos.BackupInfo.Builder; import org.apache.hadoop.hbase.protobuf.generated.BackupProtos.TableBackupStatus; +import org.apache.hadoop.hbase.util.Bytes; /** @@ -291,7 +292,15 @@ public class BackupInfo implements Comparable { this.backupStatusMap.put(table, backupStatus); } } - + + public void setTables(List tables) { + this.backupStatusMap.clear(); + for (TableName table : tables) { + BackupStatus backupStatus = new BackupStatus(table, this.targetRootDir, this.backupId); + this.backupStatusMap.put(table, backupStatus); + } + } + public String getTargetRootDir() { return targetRootDir; } @@ -365,6 +374,21 @@ public class BackupInfo implements Comparable { return builder.build(); } + @Override + public boolean equals(Object obj){ + if (obj instanceof BackupInfo) { + BackupInfo other = (BackupInfo) obj; + try { + return Bytes.equals(toByteArray(), other.toByteArray()); + } catch (IOException e) { + LOG.error(e); + return false; + } + } else { + return false; + } + } + public byte[] toByteArray() throws IOException { return toProtosBackupInfo().toByteArray(); } diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java index d6be98c..56b148e 100644 --- hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java +++ hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java @@ -280,6 +280,20 @@ public final class BackupSystemTable implements Closeable { return getBackupHistory(false); } + public ArrayList getBackupHistoryForTable(TableName name) + throws IOException + { + ArrayList history = getBackupHistory(); + ArrayList tableHistory = new ArrayList(); + for(BackupInfo info: history) { + List tables = info.getTableNames(); + if(tables.contains(name)) { + tableHistory.add(info); + } + } + return tableHistory; + } + /** * Get all backup session with a given status (in desc order by time) * @param status status @@ -455,6 +469,21 @@ public final class BackupSystemTable implements Closeable { } /** + * Removes incremental backup set + * @param backupRoot backup root + */ + + public void deleteIncrementalBackupTableSet(String backupRoot) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Delete incremental backup table set to hbase:backup. ROOT="+backupRoot); + } + try (Table table = connection.getTable(tableName)) { + Delete delete = BackupSystemTableHelper.createDeleteForIncrBackupTableSet(backupRoot); + table.delete(delete); + } + } + + /** * Register WAL files as eligible for deletion * @param files files * @param backupId backup id diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTableHelper.java hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTableHelper.java index 5eeb128..8c87cec 100644 --- hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTableHelper.java +++ hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTableHelper.java @@ -175,6 +175,17 @@ public final class BackupSystemTableHelper { } /** + * Creates Delete for incremental backup table set + * @param backupRoot backup root + * @return delete operation + */ + static Delete createDeleteForIncrBackupTableSet(String backupRoot) { + Delete delete = new Delete(rowkey(INCR_BACKUP_SET, backupRoot)); + delete.addFamily(BackupSystemTable.META_FAMILY); + return delete; + } + + /** * Creates Scan operation to load backup history * @return scan operation */ diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java index 20abba3..5c5b7d5 100644 --- hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java +++ hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java @@ -207,23 +207,23 @@ public final class BackupClientUtil { /** * Clean up the data at target directory */ - private static void cleanupTargetDir(BackupInfo backupContext, Configuration conf) { + private static void cleanupTargetDir(BackupInfo backupInfo, Configuration conf) { try { // clean up the data at target directory - LOG.debug("Trying to cleanup up target dir : " + backupContext.getBackupId()); - String targetDir = backupContext.getTargetRootDir(); + LOG.debug("Trying to cleanup up target dir : " + backupInfo.getBackupId()); + String targetDir = backupInfo.getTargetRootDir(); if (targetDir == null) { - LOG.warn("No target directory specified for " + backupContext.getBackupId()); + LOG.warn("No target directory specified for " + backupInfo.getBackupId()); return; } FileSystem outputFs = - FileSystem.get(new Path(backupContext.getTargetRootDir()).toUri(), conf); + FileSystem.get(new Path(backupInfo.getTargetRootDir()).toUri(), conf); - for (TableName table : backupContext.getTables()) { + for (TableName table : backupInfo.getTables()) { Path targetDirPath = - new Path(getTableBackupDir(backupContext.getTargetRootDir(), - backupContext.getBackupId(), table)); + new Path(getTableBackupDir(backupInfo.getTargetRootDir(), + backupInfo.getBackupId(), table)); if (outputFs.delete(targetDirPath, true)) { LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done."); } else { @@ -237,10 +237,10 @@ public final class BackupClientUtil { LOG.debug(tableDir.toString() + " is empty, remove it."); } } - outputFs.delete(new Path(targetDir, backupContext.getBackupId()), true); + outputFs.delete(new Path(targetDir, backupInfo.getBackupId()), true); } catch (IOException e1) { - LOG.error("Cleaning up backup data of " + backupContext.getBackupId() + " at " - + backupContext.getTargetRootDir() + " failed due to " + e1.getMessage() + "."); + LOG.error("Cleaning up backup data of " + backupInfo.getBackupId() + " at " + + backupInfo.getTargetRootDir() + " failed due to " + e1.getMessage() + "."); } } diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseBackupAdmin.java hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseBackupAdmin.java index 81413c6..838ebcf 100644 --- hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseBackupAdmin.java +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseBackupAdmin.java @@ -19,17 +19,25 @@ package org.apache.hadoop.hbase.client; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.Future; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupInfo; import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; import org.apache.hadoop.hbase.backup.BackupRequest; import org.apache.hadoop.hbase.backup.BackupRestoreClientFactory; +import org.apache.hadoop.hbase.backup.BackupType; import org.apache.hadoop.hbase.backup.RestoreClient; import org.apache.hadoop.hbase.backup.RestoreRequest; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; @@ -104,26 +112,227 @@ public class HBaseBackupAdmin implements BackupAdmin { @Override public int deleteBackups(String[] backupIds) throws IOException { - BackupInfo backupInfo = null; - String backupId = null; + // TODO: requires FT, failure will leave system + // in non-consistent state + int totalDeleted = 0; + Map> allTablesMap = + new HashMap>(); + try (final BackupSystemTable table = new BackupSystemTable(conn)) { - for (int i = 0; i < backupIds.length; i++) { - backupId = backupIds[i]; - LOG.info("Deleting backup for backupID=" + backupId + " ..."); - backupInfo = table.readBackupInfo(backupId); - if (backupInfo != null) { - BackupClientUtil.cleanupBackupData(backupInfo, admin.getConfiguration()); - table.deleteBackupInfo(backupInfo.getBackupId()); - LOG.info("Delete backup for backupID=" + backupId + " completed."); - totalDeleted++; - } else { - LOG.warn("Delete backup failed: no information found for backupID=" + backupId); + for (int i = 0; i < backupIds.length; i++) { + BackupInfo info = table.readBackupInfo(backupIds[i]); + if (info != null) { + String rootDir = info.getTargetRootDir(); + HashSet allTables = allTablesMap.get(rootDir); + if(allTables == null) { + allTables = new HashSet(); + allTablesMap.put(rootDir, allTables); + } + allTables.addAll(info.getTableNames()); + totalDeleted += deleteBackup(backupIds[i], table); } } + finalizeDelete(allTablesMap, table); } return totalDeleted; } + + /** + * Updates incremental backup set for every backupRoot + * @param tablesMap - Map [backupRoot: Set] + * @param table - backup system table + * @throws IOException + */ + private void finalizeDelete(Map> tablesMap, BackupSystemTable table) + throws IOException { + + for(String backupRoot: tablesMap.keySet()){ + Set incrTableSet= table.getIncrementalBackupTableSet(backupRoot); + for(TableName name: tablesMap.get(backupRoot)) { + ArrayList history = table.getBackupHistoryForTable(name); + if(history.isEmpty()) { + // No more backups for a table + incrTableSet.remove(name); + } + } + if(!incrTableSet.isEmpty()){ + table.addIncrementalBackupTableSet(incrTableSet, backupRoot); + } else { // empty + table.deleteIncrementalBackupTableSet(backupRoot); + } + } + } + + + /** + * Delete single backup and all related backups + * @param backupId - backup id + * @return total number of deleted backup images + * @throws IOException + */ + private int deleteBackup(String backupId, BackupSystemTable table) throws IOException { + + BackupInfo backupInfo = table.readBackupInfo(backupId); + + int totalDeleted = 0; + if (backupInfo != null) { + LOG.info("Deleting backup " + backupInfo.getBackupId() + " ..."); + BackupClientUtil.cleanupBackupData(backupInfo, admin.getConfiguration()); + // List of tables in this backup; + List tables = backupInfo.getTableNames(); + long startTime = backupInfo.getStartTs(); + for(TableName tn: tables){ + boolean isLastBackupSession = isLastBackupSession(table, tn, startTime); + + // Backup type: FULL or INCREMENTAL + // Last backup session for T: YES or NO + // ALgorithm: + // For every table T from table list 'tables': + // if(FULL, YES) deletes only physical data (PD) + // if(FULL, NO), deletes PD, scans all newer backups and removes T from backupInfo, until + // we either reach the most recent backup for T in the system or FULL backup which + // includes T + // if(INCREMENTAL, YES) deletes only physical data (PD) + // if(INCREMENTAL, NO) deletes physical data and for table T scans all backup images + // between last FULL backup, which is older than the backup being deleted and the next + // FULL backup (if exists) or last one for a particular table T and removes T from list + // of backup tables. + if(isLastBackupSession) { + continue; + } + // else + List affectedBackups = getAffectedBackupInfos(backupInfo, tn, table); + for (BackupInfo info: affectedBackups) { + if(info.equals(backupInfo)) { + continue; + } + removeTableFromBackupImage(info, tn, table); + } + } + table.deleteBackupInfo(backupInfo.getBackupId()); + LOG.info("Delete backup " + backupInfo.getBackupId() + " completed."); + totalDeleted++; + } else { + LOG.warn("Delete backup failed: no information found for backupID=" + backupId); + } + return totalDeleted; + } + + + private void removeTableFromBackupImage(BackupInfo info, TableName tn, + BackupSystemTable table) throws IOException { + List tables = info.getTableNames(); + if(tables.contains(tn)) { + tables.remove(tn); + + if(tables.isEmpty()) { + table.deleteBackupInfo(info.getBackupId()); + BackupClientUtil.cleanupBackupData(info, conn.getConfiguration()); + } else { + info.setTables(tables); + table.updateBackupInfo(info); + // Now, clean up directory for table + cleanupBackupDir(info, tn, conn.getConfiguration()); + } + } + } + + + private List getAffectedBackupInfos(BackupInfo backupInfo, + TableName tn, BackupSystemTable table) throws IOException { + long ts = backupInfo.getStartTs(); + BackupType type = backupInfo.getType(); + List list = new ArrayList(); + List history = table.getBackupHistory(); + if(type == BackupType.FULL) { + // Scan from most recent to backupInfo + // break when backupInfo reached + for(BackupInfo info: history){ + if(info.getStartTs() == ts) { + break; + } + List tables = info.getTableNames(); + if(tables.contains(tn)) { + BackupType bt = info.getType(); + if(bt == BackupType.FULL) { + list.clear(); + } else{ + list.add(info); + } + } + } + } else{ + // Find first FULL backup image which contains + // 'tn' and which is older than 'backupInfo' + // + // it can return null? + for(BackupInfo info: history){ + List tables = info.getTableNames(); + if(info.getStartTs() == ts) { + break; + } + if(tables.contains(tn)) { + BackupType bt = info.getType(); + if(bt == BackupType.FULL) { + list.clear(); + } else{ + list.add(info); + } + } + } + } + return list; + } + + + /** + * Clean up the data at target directory + */ + private void cleanupBackupDir(BackupInfo backupInfo, TableName table, Configuration conf) { + try { + // clean up the data at target directory + String targetDir = backupInfo.getTargetRootDir(); + if (targetDir == null) { + LOG.warn("No target directory specified for " + backupInfo.getBackupId()); + return; + } + + FileSystem outputFs = + FileSystem.get(new Path(backupInfo.getTargetRootDir()).toUri(), conf); + + Path targetDirPath = + new Path(BackupClientUtil.getTableBackupDir(backupInfo.getTargetRootDir(), + backupInfo.getBackupId(), table)); + if (outputFs.delete(targetDirPath, true)) { + LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done."); + } else { + LOG.info("No data has been found in " + targetDirPath.toString() + "."); + } + + } catch (IOException e1) { + LOG.error("Cleaning up backup data of " + backupInfo.getBackupId() + " for table "+ table + +"at " + backupInfo.getTargetRootDir() + " failed due to " + e1.getMessage() + "."); + } + } + + private boolean isLastBackupSession(BackupSystemTable table, TableName tn, long startTime) + throws IOException { + ArrayList history = table.getBackupHistory(); + for(BackupInfo info: history){ + List tables = info.getTableNames(); + if (!tables.contains(tn)) { + continue; + } + if(info.getStartTs() <= startTime) { + return true; + } else { + return false; + } + } + return false; + } + @Override public List getHistory(int n) throws IOException { diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMultipleDeletes.java hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMultipleDeletes.java new file mode 100644 index 0000000..5fc671d --- /dev/null +++ hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMultipleDeletes.java @@ -0,0 +1,195 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.hamcrest.CoreMatchers; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Lists; + + +/** + * Create multiple backups for two tables: table1, table2 + * then perform 3 deletes + */ +@Category(LargeTests.class) +public class TestBackupMultipleDeletes extends TestBackupBase { + private static final Log LOG = LogFactory.getLog(TestBackupMultipleDeletes.class); + @Test + public void testBackupMultipleDeletes() throws Exception { + // #1 - create full backup for all tables + LOG.info("create full backup image for all tables"); + + List tables = Lists.newArrayList(table1, table2); + HBaseAdmin admin = null; + Connection conn = ConnectionFactory.createConnection(conf1); + admin = (HBaseAdmin) conn.getAdmin(); + + BackupRequest request = new BackupRequest(); + request.setBackupType(BackupType.FULL).setTableList(tables).setTargetRootDir(BACKUP_ROOT_DIR); + String backupIdFull = admin.getBackupAdmin().backupTables(request); + + assertTrue(checkSucceeded(backupIdFull)); + + // #2 - insert some data to table table1 + HTable t1 = (HTable) conn.getTable(table1); + Put p1; + for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { + p1 = new Put(Bytes.toBytes("row-t1" + i)); + p1.addColumn(famName, qualName, Bytes.toBytes("val" + i)); + t1.put(p1); + } + + Assert.assertThat(TEST_UTIL.countRows(t1), CoreMatchers.equalTo(NB_ROWS_IN_BATCH * 2)); + t1.close(); + + + // #3 - incremental backup for table1, table2 + tables = Lists.newArrayList(table1, table2); + request = new BackupRequest(); + request.setBackupType(BackupType.INCREMENTAL).setTableList(tables) + .setTargetRootDir(BACKUP_ROOT_DIR); + String backupIdInc1 = admin.getBackupAdmin().backupTables(request); + assertTrue(checkSucceeded(backupIdInc1)); + + // #4 - insert some data to table table2 + HTable t2 = (HTable) conn.getTable(table2); + Put p2 = null; + for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { + p2 = new Put(Bytes.toBytes("row-t2" + i)); + p2.addColumn(famName, qualName, Bytes.toBytes("val" + i)); + t2.put(p2); + } + + + // #5 - incremental backup for table1, table2 + tables = Lists.newArrayList(table1, table2); + request = new BackupRequest(); + request.setBackupType(BackupType.INCREMENTAL).setTableList(tables) + .setTargetRootDir(BACKUP_ROOT_DIR); + String backupIdInc2 = admin.getBackupAdmin().backupTables(request); + assertTrue(checkSucceeded(backupIdInc2)); + + // #6 - insert some data to table table1 + t1 = (HTable) conn.getTable(table1); + for (int i = NB_ROWS_IN_BATCH; i < 2*NB_ROWS_IN_BATCH; i++) { + p1 = new Put(Bytes.toBytes("row-t1" + i)); + p1.addColumn(famName, qualName, Bytes.toBytes("val" + i)); + t1.put(p1); + } + + + // #7 - incremental backup for table1, table2 + tables = Lists.newArrayList(table1, table2); + request = new BackupRequest(); + request.setBackupType(BackupType.INCREMENTAL).setTableList(tables) + .setTargetRootDir(BACKUP_ROOT_DIR); + String backupIdInc3 = admin.getBackupAdmin().backupTables(request); + assertTrue(checkSucceeded(backupIdInc3)); + + // #8 - insert some data to table table2 + t2 = (HTable) conn.getTable(table2); + for (int i = NB_ROWS_IN_BATCH; i < 2*NB_ROWS_IN_BATCH; i++) { + p2 = new Put(Bytes.toBytes("row-t1" + i)); + p2.addColumn(famName, qualName, Bytes.toBytes("val" + i)); + t2.put(p2); + } + + // #9 - incremental backup for table1, table2 + tables = Lists.newArrayList(table1, table2); + request = new BackupRequest(); + request.setBackupType(BackupType.INCREMENTAL).setTableList(tables) + .setTargetRootDir(BACKUP_ROOT_DIR); + String backupIdInc4 = admin.getBackupAdmin().backupTables(request); + assertTrue(checkSucceeded(backupIdInc4)); + + + // #10 - insert some data to table table1 + t1 = (HTable) conn.getTable(table1); + for (int i = 2*NB_ROWS_IN_BATCH; i < 3*NB_ROWS_IN_BATCH; i++) { + p1 = new Put(Bytes.toBytes("row-t1" + i)); + p1.addColumn(famName, qualName, Bytes.toBytes("val" + i)); + t1.put(p1); + } + + // #11- incremental backup for table1, table2 + tables = Lists.newArrayList(table1, table2); + request = new BackupRequest(); + request.setBackupType(BackupType.INCREMENTAL).setTableList(tables) + .setTargetRootDir(BACKUP_ROOT_DIR); + String backupIdInc5 = admin.getBackupAdmin().backupTables(request); + assertTrue(checkSucceeded(backupIdInc5)); + + // #12 - insert some data to table table2 + t2 = (HTable) conn.getTable(table2); + for (int i = 2*NB_ROWS_IN_BATCH; i < 3*NB_ROWS_IN_BATCH; i++) { + p2 = new Put(Bytes.toBytes("row-t1" + i)); + p2.addColumn(famName, qualName, Bytes.toBytes("val" + i)); + t2.put(p2); + } + + // #13- incremental backup for table1, table2 + tables = Lists.newArrayList(table1, table2); + request = new BackupRequest(); + request.setBackupType(BackupType.INCREMENTAL).setTableList(tables) + .setTargetRootDir(BACKUP_ROOT_DIR); + String backupIdInc6 = admin.getBackupAdmin().backupTables(request); + assertTrue(checkSucceeded(backupIdInc6)); + + int totalBackups = admin.getBackupAdmin().getHistory(100).size(); + + LOG.error("Delete backupIdInc6"); + admin.getBackupAdmin().deleteBackups( new String[]{backupIdInc6}); + LOG.error("Delete backupIdInc6 done"); + int backups = admin.getBackupAdmin().getHistory(100).size(); + assertEquals(totalBackups -1, backups); + LOG.error("Delete backupIdInc3"); + admin.getBackupAdmin().deleteBackups( new String[]{backupIdInc3}); + LOG.error("Delete backupIdInc3 done"); + backups = admin.getBackupAdmin().getHistory(100).size(); + assertEquals(totalBackups - 4, backups); + LOG.error("Delete backupIdFull"); + admin.getBackupAdmin().deleteBackups( new String[]{backupIdFull}); + LOG.error("Delete backupIdFull done"); + backups = admin.getBackupAdmin().getHistory(100).size(); + + assertEquals(totalBackups - 7, backups); + + admin.close(); + conn.close(); + } + +}