From 260103377b870795ab47f5a66268b119a7c210c4 Mon Sep 17 00:00:00 2001 From: Vladimir Rodionov Date: Wed, 6 Jun 2018 15:37:39 -0700 Subject: [PATCH] HBASE-20630: B&R: Delete command enhancements --- .../apache/hadoop/hbase/backup/BackupDriver.java | 11 ++- .../hbase/backup/BackupRestoreConstants.java | 7 +- .../hadoop/hbase/backup/impl/BackupCommands.java | 85 +++++++++++++++++++--- .../hadoop/hbase/backup/TestBackupDelete.java | 58 ++++++++++++++- 4 files changed, 147 insertions(+), 14 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java index 8baf2f0b32..76c412c6ef 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java @@ -17,12 +17,16 @@ */ package org.apache.hadoop.hbase.backup; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BACKUP_LIST_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_LIST; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET; @@ -46,14 +50,13 @@ import org.apache.hadoop.hbase.backup.impl.BackupManager; import org.apache.hadoop.hbase.util.AbstractHBaseTool; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.util.ToolRunner; +import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; - /** * * Command-line entry point for backup operation @@ -152,10 +155,12 @@ public class BackupDriver extends AbstractHBaseTool { addOptNoArg(OPTION_DEBUG, OPTION_DEBUG_DESC); addOptWithArg(OPTION_TABLE, OPTION_TABLE_DESC); addOptWithArg(OPTION_BANDWIDTH, OPTION_BANDWIDTH_DESC); + addOptWithArg(OPTION_LIST, OPTION_BACKUP_LIST_DESC); addOptWithArg(OPTION_WORKERS, OPTION_WORKERS_DESC); addOptWithArg(OPTION_RECORD_NUMBER, OPTION_RECORD_NUMBER_DESC); addOptWithArg(OPTION_SET, OPTION_SET_DESC); addOptWithArg(OPTION_PATH, OPTION_PATH_DESC); + addOptWithArg(OPTION_KEEP, OPTION_KEEP_DESC); addOptWithArg(OPTION_YARN_QUEUE_NAME, OPTION_YARN_QUEUE_NAME_DESC); } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java index eaeef22d7e..16ec3d2030 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java @@ -65,8 +65,9 @@ public interface BackupRestoreConstants { String OPTION_TABLE_DESC = "Table name. If specified, only backup images," + " which contain this table will be listed."; - String OPTION_TABLE_LIST = "l"; + String OPTION_LIST = "l"; String OPTION_TABLE_LIST_DESC = "Table name list, comma-separated."; + String OPTION_BACKUP_LIST_DESC = "Backup ids list, comma-separated."; String OPTION_BANDWIDTH = "b"; String OPTION_BANDWIDTH_DESC = "Bandwidth per task (MapReduce task) in MB/s"; @@ -80,6 +81,10 @@ public interface BackupRestoreConstants { String OPTION_PATH = "p"; String OPTION_PATH_DESC = "Backup destination root directory path"; + String OPTION_KEEP = "k"; + String OPTION_KEEP_DESC = "Specifies maximum age of backup (in days) to keep during bulk delete"; + + String OPTION_TABLE_MAPPING = "m"; String OPTION_TABLE_MAPPING_DESC = "A comma separated list of target tables. " diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java index 3e28f51a7c..f78ab01cdf 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java @@ -18,10 +18,14 @@ package org.apache.hadoop.hbase.backup.impl; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BACKUP_LIST_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_LIST; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER; @@ -61,11 +65,11 @@ import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.yetus.audience.InterfaceAudience; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter; import org.apache.hbase.thirdparty.org.apache.commons.cli.Options; +import org.apache.yetus.audience.InterfaceAudience; /** * General backup commands, options and usage messages @@ -105,8 +109,7 @@ public final class BackupCommands { public static final String HISTORY_CMD_USAGE = "Usage: hbase backup history [options]"; - public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete \n" - + " backup_id Backup image id\n"; + public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete [options]"; public static final String REPAIR_CMD_USAGE = "Usage: hbase backup repair\n"; @@ -560,29 +563,93 @@ public final class BackupCommands { @Override public void execute() throws IOException { - if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { + + if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 1) { printUsage(); throw new IOException(INCORRECT_USAGE); } + if (!cmdline.hasOption(OPTION_PATH) && !cmdline.hasOption(OPTION_LIST)) { + printUsage(); + throw new IOException(INCORRECT_USAGE); + } super.execute(); + if (cmdline.hasOption(OPTION_PATH)) { + executeDeleteOlderThan(cmdline); + } else if (cmdline.hasOption(OPTION_LIST)) { + executeDeleteListOfBackups(cmdline); + } + } + + private void executeDeleteOlderThan(CommandLine cmdline) throws IOException { + String value = cmdline.getOptionValue(OPTION_PATH); + int days = 0; + try { + days = Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new IOException(value + " is not an integer number"); + } + final long fdays = days; + BackupInfo.Filter dateFilter = new BackupInfo.Filter() { + @Override + public boolean apply(BackupInfo info) { + long currentTime = EnvironmentEdgeManager.currentTime(); + long maxTsToDelete = currentTime - fdays * 24 * 3600 * 1000; + return info.getCompleteTs() <= maxTsToDelete; + } + }; + List history = null; + try (final BackupSystemTable sysTable = new BackupSystemTable(conn); + BackupAdminImpl admin = new BackupAdminImpl(conn)) { + history = sysTable.getBackupHistory(-1, dateFilter); + String[] backupIds = convertToBackupIds(history); + int deleted = admin.deleteBackups(backupIds); + System.out.println("Deleted " + deleted + " backups. Total older than " + days + " days: " + + backupIds.length); + } catch (IOException e) { + System.err.println("Delete command FAILED. Please run backup repair tool to restore backup " + + "system integrity"); + throw e; + } + } + + private String[] convertToBackupIds(List history) { + String[] ids = new String[history.size()]; + for (int i = 0; i < ids.length; i++) { + ids[i] = history.get(i).getBackupId(); + } + return ids; + } + + private void executeDeleteListOfBackups(CommandLine cmdline) throws IOException { + String value = cmdline.getOptionValue(OPTION_LIST); + String[] backupIds = value.split(","); - String[] args = cmdline.getArgs(); - String[] backupIds = new String[args.length - 1]; - System.arraycopy(args, 1, backupIds, 0, backupIds.length); try (BackupAdminImpl admin = new BackupAdminImpl(conn)) { int deleted = admin.deleteBackups(backupIds); - System.out.println("Deleted " + deleted + " backups. Total requested: " + (args.length -1)); + System.out.println("Deleted " + deleted + " backups. Total requested: " + backupIds.length); } catch (IOException e) { System.err.println("Delete command FAILED. Please run backup repair tool to restore backup " - + "system integrity"); + + "system integrity"); throw e; } + } @Override protected void printUsage() { System.out.println(DELETE_CMD_USAGE); + Options options = new Options(); + options.addOption(OPTION_KEEP, true, OPTION_KEEP_DESC); + options.addOption(OPTION_LIST, true, OPTION_BACKUP_LIST_DESC); + + HelpFormatter helpFormatter = new HelpFormatter(); + helpFormatter.setLeftPadding(2); + helpFormatter.setDescPadding(8); + helpFormatter.setWidth(100); + helpFormatter.setSyntaxPrefix("Options:"); + helpFormatter.printHelp(" ", null, options, USAGE_FOOTER); + } } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java index 9e67ff731e..c333578904 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java @@ -28,7 +28,10 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.EnvironmentEdge; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.util.ToolRunner; +import org.junit.Assert; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -90,7 +93,7 @@ public class TestBackupDelete extends TestBackupBase { ByteArrayOutputStream baos = new ByteArrayOutputStream(); System.setOut(new PrintStream(baos)); - String[] args = new String[] { "delete", backupId }; + String[] args = new String[] { "delete", "-l", backupId }; // Run backup try { @@ -104,4 +107,57 @@ public class TestBackupDelete extends TestBackupBase { LOG.info(baos.toString()); assertTrue(output.indexOf("Deleted 1 backups") >= 0); } + + @Test + public void testBackupPurgeOldBackupsCommand() throws Exception { + LOG.info("test backup delete (purge old backups) on a single table with data: command-line"); + List tableList = Lists.newArrayList(table1); + EnvironmentEdgeManager.injectEdge(new EnvironmentEdge() { + // time - 2 days + @Override + public long currentTime() { + return System.currentTimeMillis() - 2 * 24 * 3600 * 1000 ; + } + }); + String backupId = fullTableBackup(tableList); + assertTrue(checkSucceeded(backupId)); + + EnvironmentEdgeManager.reset(); + + LOG.info("backup complete"); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + // Purge all backups which are older than 3 days + // Must return 0 (no backups were purged) + String[] args = new String[] { "delete", "-p", "3" }; + // Run backup + + try { + int ret = ToolRunner.run(conf1, new BackupDriver(), args); + assertTrue(ret == 0); + } catch (Exception e) { + LOG.error("failed", e); + Assert.fail(e.getMessage()); + } + String output = baos.toString(); + LOG.info(baos.toString()); + assertTrue(output.indexOf("Deleted 0 backups") >= 0); + + // Purge all backups which are older than 1 days + // Must return 1 deleted backup + args = new String[] { "delete", "-p", "1" }; + // Run backup + baos.reset(); + try { + int ret = ToolRunner.run(conf1, new BackupDriver(), args); + assertTrue(ret == 0); + } catch (Exception e) { + LOG.error("failed", e); + Assert.fail(e.getMessage()); + } + output = baos.toString(); + LOG.info(baos.toString()); + assertTrue(output.indexOf("Deleted 1 backups") >= 0); + } } -- 2.14.3 (Apple Git-98)