diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java index d1ab246..80f022f 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java @@ -45,8 +45,6 @@ public interface BackupRestoreConstants { public static final String BACKUP_ATTEMPTS_PAUSE_MS_KEY = "hbase.backup.attempts.pause.ms"; public static final int DEFAULT_BACKUP_ATTEMPTS_PAUSE_MS = 10000; - - /* * Drivers option list */ diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java index 75e0ab7..7ea923e 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java @@ -53,8 +53,8 @@ import org.apache.hadoop.hbase.backup.BackupRequest; import org.apache.hadoop.hbase.backup.BackupRestoreConstants; import org.apache.hadoop.hbase.backup.BackupRestoreConstants.BackupCommand; import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.backup.util.BackupSet; +import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; @@ -114,9 +114,12 @@ public final class BackupCommands { public static abstract class Command extends Configured { CommandLine cmdline; - + Connection conn; Command(Configuration conf) { - super(conf); + if (conf == null) { + conf = HBaseConfiguration.create(); + } + setConf(conf); } public void execute() throws IOException { @@ -124,9 +127,34 @@ public final class BackupCommands { printUsage(); throw new IOException(INCORRECT_USAGE); } + + // Create connection + conn = ConnectionFactory.createConnection(getConf()); + if (requiresNoActiveSession()) { + // Check active session + try (BackupSystemTable table = new BackupSystemTable(conn);) { + List sessions = table.getBackupInfos(BackupState.RUNNING); + + if(sessions.size() > 0) { + System.err.println("Found backup session in a RUNNING state: "); + System.err.println(sessions.get(0)); + System.err.println("This may indicate that a previous session has failed abnormally."); + System.err.println("In this case, backup recovery is recommended."); + throw new IOException("Active session found, aborted command execution"); + } + } + } } protected abstract void printUsage(); + + /** + * The command can't be run if active backup session is in progress + * @return true if no active sessions are in progress + */ + protected boolean requiresNoActiveSession() { + return false; + } } private BackupCommands() { @@ -178,8 +206,12 @@ public final class BackupCommands { } @Override + protected boolean requiresNoActiveSession() { + return true; + } + + @Override public void execute() throws IOException { - super.execute(); if (cmdline == null || cmdline.getArgs() == null) { printUsage(); throw new IOException(INCORRECT_USAGE); @@ -202,8 +234,8 @@ public final class BackupCommands { throw new IOException(INCORRECT_USAGE); } + String tables = null; - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); // Check if we have both: backup set and list of tables if (cmdline.hasOption(OPTION_TABLE) && cmdline.hasOption(OPTION_SET)) { @@ -212,12 +244,13 @@ public final class BackupCommands { printUsage(); throw new IOException(INCORRECT_USAGE); } - + // Creates connection + super.execute(); // Check backup set String setName = null; if (cmdline.hasOption(OPTION_SET)) { setName = cmdline.getOptionValue(OPTION_SET); - tables = getTablesForSet(setName, conf); + tables = getTablesForSet(setName, getConf()); if (tables == null) { System.out.println("ERROR: Backup set '" + setName @@ -235,8 +268,7 @@ public final class BackupCommands { cmdline.hasOption(OPTION_WORKERS) ? Integer.parseInt(cmdline .getOptionValue(OPTION_WORKERS)) : -1; - try (Connection conn = ConnectionFactory.createConnection(getConf()); - BackupAdminImpl admin = new BackupAdminImpl(conn);) { + try (BackupAdminImpl admin = new BackupAdminImpl(conn);) { BackupRequest.Builder builder = new BackupRequest.Builder(); BackupRequest request = builder.withBackupType(BackupType.valueOf(args[1].toUpperCase())) @@ -268,8 +300,7 @@ public final class BackupCommands { } private String getTablesForSet(String name, Configuration conf) throws IOException { - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable table = new BackupSystemTable(conn)) { + try (final BackupSystemTable table = new BackupSystemTable(conn)) { List tables = table.describeBackupSet(name); if (tables == null) return null; return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND); @@ -304,7 +335,6 @@ public final class BackupCommands { @Override public void execute() throws IOException { - super.execute(); if (cmdline == null) { printUsage(); throw new IOException(INCORRECT_USAGE); @@ -359,7 +389,6 @@ public final class BackupCommands { @Override public void execute() throws IOException { - super.execute(); if (cmdline == null || cmdline.getArgs() == null) { printUsage(); throw new IOException(INCORRECT_USAGE); @@ -370,10 +399,10 @@ public final class BackupCommands { throw new IOException(INCORRECT_USAGE); } + super.execute(); + String backupId = args[1]; - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable sysTable = new BackupSystemTable(conn);) { + try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) { BackupInfo info = sysTable.readBackupInfo(backupId); if (info == null) { System.out.println("ERROR: " + backupId + " does not exist"); @@ -399,7 +428,6 @@ public final class BackupCommands { @Override public void execute() throws IOException { - super.execute(); if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length == 1) { System.out.println("No backup id was specified, " @@ -412,10 +440,10 @@ public final class BackupCommands { throw new IOException(INCORRECT_USAGE); } + super.execute(); + String backupId = (args == null || args.length <= 1) ? null : args[1]; - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable sysTable = new BackupSystemTable(conn);) { + try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) { BackupInfo info = null; if (backupId != null) { @@ -456,19 +484,23 @@ public final class BackupCommands { } @Override + protected boolean requiresNoActiveSession() { + return true; + } + + @Override public void execute() throws IOException { - super.execute(); if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { printUsage(); throw new IOException(INCORRECT_USAGE); } + super.execute(); + String[] args = cmdline.getArgs(); String[] backupIds = new String[args.length - 1]; System.arraycopy(args, 1, backupIds, 0, backupIds.length); - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - try (final Connection conn = ConnectionFactory.createConnection(conf); - BackupAdminImpl admin = new BackupAdminImpl(conn);) { + try (BackupAdminImpl admin = new BackupAdminImpl(conn);) { int deleted = admin.deleteBackups(backupIds); System.out.println("Deleted " + deleted + " backups. Total requested: " + args.length); } @@ -512,7 +544,6 @@ public final class BackupCommands { @Override public void execute() throws IOException { - super.execute(); int n = parseHistoryLength(); final TableName tableName = getTableName(); @@ -535,18 +566,16 @@ public final class BackupCommands { }; Path backupRootPath = getBackupRootPath(); List history = null; - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); if (backupRootPath == null) { // Load from backup system table - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable sysTable = new BackupSystemTable(conn);) { - + super.execute(); + try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) { history = sysTable.getBackupHistory(n, tableNameFilter, tableSetFilter); } } else { // load from backup FS history = - BackupUtils.getHistory(conf, n, backupRootPath, tableNameFilter, tableSetFilter); + BackupUtils.getHistory(getConf(), n, backupRootPath, tableNameFilter, tableSetFilter); } for (BackupInfo info : history) { System.out.println(info.getShortDescription()); @@ -627,7 +656,6 @@ public final class BackupCommands { @Override public void execute() throws IOException { - super.execute(); // Command-line must have at least one element if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { printUsage(); @@ -661,11 +689,11 @@ public final class BackupCommands { } private void processSetList(String[] args) throws IOException { + super.execute(); + // List all backup set names // does not expect any args - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - try (final Connection conn = ConnectionFactory.createConnection(conf); - BackupAdminImpl admin = new BackupAdminImpl(conn);) { + try (BackupAdminImpl admin = new BackupAdminImpl(conn);) { List list = admin.listBackupSets(); for (BackupSet bs : list) { System.out.println(bs); @@ -678,10 +706,10 @@ public final class BackupCommands { printUsage(); throw new IOException(INCORRECT_USAGE); } + super.execute(); + String setName = args[2]; - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable sysTable = new BackupSystemTable(conn);) { + try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) { List tables = sysTable.describeBackupSet(setName); BackupSet set = tables == null ? null : new BackupSet(setName, tables); if (set == null) { @@ -697,10 +725,10 @@ public final class BackupCommands { printUsage(); throw new IOException(INCORRECT_USAGE); } + super.execute(); + String setName = args[2]; - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupAdminImpl admin = new BackupAdminImpl(conn);) { + try (final BackupAdminImpl admin = new BackupAdminImpl(conn);) { boolean result = admin.deleteBackupSet(setName); if (result) { System.out.println("Delete set " + setName + " OK."); @@ -715,13 +743,12 @@ public final class BackupCommands { printUsage(); throw new IOException(INCORRECT_USAGE); } + super.execute(); String setName = args[2]; String[] tables = args[3].split(","); TableName[] tableNames = toTableNames(tables); - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupAdminImpl admin = new BackupAdminImpl(conn);) { + try (final BackupAdminImpl admin = new BackupAdminImpl(conn);) { admin.removeFromBackupSet(setName, tableNames); } } @@ -739,15 +766,15 @@ public final class BackupCommands { printUsage(); throw new IOException(INCORRECT_USAGE); } + super.execute(); + String setName = args[2]; String[] tables = args[3].split(","); TableName[] tableNames = new TableName[tables.length]; for (int i = 0; i < tables.length; i++) { tableNames[i] = TableName.valueOf(tables[i]); } - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupAdminImpl admin = new BackupAdminImpl(conn);) { + try (final BackupAdminImpl admin = new BackupAdminImpl(conn);) { admin.addToBackupSet(setName, tableNames); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java index 217e750..870aca9 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java @@ -1302,9 +1302,9 @@ public final class BackupSystemTable implements Closeable { return getTableName(conf).getNameAsString(); } - - - + public static String getSnapshotName(Configuration conf) { + return "snapshot_"+getTableNameAsString(conf).replace(":", "_"); + } /** * Creates Put operation for a given backup info object diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java index 125b5da..dbe2ef5 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java @@ -88,6 +88,9 @@ public abstract class TableBackupClient { */ protected void beginBackup(BackupManager backupManager, BackupInfo backupInfo) throws IOException { + + snapshotBackupTable(); + backupManager.setBackupInfo(backupInfo); // set the start timestamp of the overall backup long startTs = EnvironmentEdgeManager.currentTime(); @@ -116,7 +119,7 @@ public abstract class TableBackupClient { * @param backupInfo backup info * @throws Exception exception */ - private void deleteSnapshot(final Connection conn, BackupInfo backupInfo, Configuration conf) + private static void deleteSnapshots(final Connection conn, BackupInfo backupInfo, Configuration conf) throws IOException { LOG.debug("Trying to delete snapshot for full backup."); for (String snapshotName : backupInfo.getSnapshotNames()) { @@ -127,8 +130,6 @@ public abstract class TableBackupClient { try (Admin admin = conn.getAdmin();) { admin.deleteSnapshot(snapshotName); - } catch (IOException ioe) { - LOG.debug("when deleting snapshot " + snapshotName, ioe); } LOG.debug("Deleting the snapshot " + snapshotName + " for backup " + backupInfo.getBackupId() + " succeeded."); @@ -140,7 +141,7 @@ public abstract class TableBackupClient { * snapshots. * @throws IOException exception */ - private void cleanupExportSnapshotLog(Configuration conf) throws IOException { + private static void cleanupExportSnapshotLog(Configuration conf) throws IOException { FileSystem fs = FSUtils.getCurrentFileSystem(conf); Path stagingDir = new Path(conf.get(BackupRestoreConstants.CONF_STAGING_ROOT, fs.getWorkingDirectory() @@ -163,7 +164,7 @@ public abstract class TableBackupClient { * Clean up the uncompleted data at target directory if the ongoing backup has already entered * the copy phase. */ - private void cleanupTargetDir(BackupInfo backupInfo, Configuration conf) { + private static void cleanupTargetDir(BackupInfo backupInfo, Configuration conf) { try { // clean up the uncompleted data at target directory if the ongoing backup has already entered // the copy phase @@ -229,21 +230,68 @@ public abstract class TableBackupClient { + ",failedts=" + backupInfo.getCompleteTs() + ",failedphase=" + backupInfo.getPhase() + ",failedmessage=" + backupInfo.getFailedMsg(); LOG.error(backupFailedData); - + cleanupAndRestoreBackupSystem(conn, backupInfo, conf); + // If backup session is updated to FAILED state - means we + // processed recovery already. backupManager.updateBackupInfo(backupInfo); - // if full backup, then delete HBase snapshots if there already are snapshots taken - // and also clean up export snapshot log files if exist - if (type == BackupType.FULL) { - deleteSnapshot(conn, backupInfo, conf); - cleanupExportSnapshotLog(conf); + LOG.info("Backup " + backupInfo.getBackupId() + " failed."); + } + + public static void cleanupAndRestoreBackupSystem (Connection conn, BackupInfo backupInfo, + Configuration conf) throws IOException + { + BackupType type = backupInfo.getType(); + // if full backup, then delete HBase snapshots if there already are snapshots taken + // and also clean up export snapshot log files if exist + if (type == BackupType.FULL) { + deleteSnapshots(conn, backupInfo, conf); + cleanupExportSnapshotLog(conf); + } + restoreBackupTable(conn, conf); + deleteBackupTableSnapshot(conn, conf); + // clean up the uncompleted data at target directory if the ongoing backup has already entered + // the copy phase + // For incremental backup, DistCp logs will be cleaned with the targetDir. + cleanupTargetDir(backupInfo, conf); + } + + private void snapshotBackupTable() throws IOException { + + try (Admin admin = conn.getAdmin();){ + admin.snapshot(BackupSystemTable.getSnapshotName(conf), + BackupSystemTable.getTableName(conf)); } + } - // clean up the uncompleted data at target directory if the ongoing backup has already entered - // the copy phase - // For incremental backup, DistCp logs will be cleaned with the targetDir. - cleanupTargetDir(backupInfo, conf); - LOG.info("Backup " + backupInfo.getBackupId() + " failed."); + private static void restoreBackupTable(Connection conn, Configuration conf) + throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Restoring " + BackupSystemTable.getTableNameAsString(conf) + + " from snapshot"); + } + try (Admin admin = conn.getAdmin();) { + String snapshotName = BackupSystemTable.getSnapshotName(conf); + admin.restoreSnapshot(snapshotName); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Done restoring backup system table"); + } + } + + private static void deleteBackupTableSnapshot(Connection conn, Configuration conf) + throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Deleting " + BackupSystemTable.getSnapshotName(conf) + + " from the system"); + } + try (Admin admin = conn.getAdmin();) { + String snapshotName = BackupSystemTable.getSnapshotName(conf); + admin.deleteSnapshot(snapshotName); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Done deleting backup system table snapshot"); + } } /** @@ -366,11 +414,12 @@ public abstract class TableBackupClient { // - clean up directories with prefix "exportSnapshot-", which are generated when exporting // snapshots if (type == BackupType.FULL) { - deleteSnapshot(conn, backupInfo, conf); + deleteSnapshots(conn, backupInfo, conf); cleanupExportSnapshotLog(conf); } else if (type == BackupType.INCREMENTAL) { cleanupDistCpLog(backupInfo, conf); } + deleteBackupTableSnapshot(conn, conf); LOG.info("Backup " + backupInfo.getBackupId() + " completed."); }