diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java index 3fd0c33..32ef218 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java @@ -125,37 +125,37 @@ public class RestoreDriver extends AbstractHBaseTool { String backupId = remainArgs[1]; String tables = null; String tableMapping = null; - // Check backup set - if (cmd.hasOption(OPTION_SET)) { - String setName = cmd.getOptionValue(OPTION_SET); - try{ - tables = getTablesForSet(setName, conf); - } catch(IOException e){ - System.out.println("ERROR: "+ e.getMessage()+" for setName="+setName); - return -2; - } - if (tables == null) { - System.out.println("ERROR: Backup set '" + setName - + "' is either empty or does not exist"); - return -3; - } - tableMapping = (remainArgs.length > 2) ? remainArgs[2] : null; - } else { - tables = remainArgs[2]; - tableMapping = (remainArgs.length > 3) ? remainArgs[3] : null; - } - - TableName[] sTableArray = BackupServerUtil.parseTableNames(tables); - TableName[] tTableArray = BackupServerUtil.parseTableNames(tableMapping); - - if (sTableArray != null && tTableArray != null && (sTableArray.length != tTableArray.length)) { - System.out.println("ERROR: table mapping mismatch: " + tables + " : " + tableMapping); - System.out.println(USAGE); - return -4; - } - try (final Connection conn = ConnectionFactory.createConnection(conf); BackupAdmin client = conn.getAdmin().getBackupAdmin();) { + // Check backup set + if (cmd.hasOption(OPTION_SET)) { + String setName = cmd.getOptionValue(OPTION_SET); + try{ + tables = getTablesForSet(conn, setName, conf); + } catch(IOException e){ + System.out.println("ERROR: "+ e.getMessage()+" for setName="+setName); + return -2; + } + if (tables == null) { + System.out.println("ERROR: Backup set '" + setName + + "' is either empty or does not exist"); + return -3; + } + tableMapping = (remainArgs.length > 2) ? remainArgs[2] : null; + } else { + tables = remainArgs[2]; + tableMapping = (remainArgs.length > 3) ? remainArgs[3] : null; + } + + TableName[] sTableArray = BackupServerUtil.parseTableNames(tables); + TableName[] tTableArray = BackupServerUtil.parseTableNames(tableMapping); + + if (sTableArray != null && tTableArray != null && (sTableArray.length != tTableArray.length)){ + System.out.println("ERROR: table mapping mismatch: " + tables + " : " + tableMapping); + System.out.println(USAGE); + return -4; + } + client.restore(RestoreServerUtil.createRestoreRequest(backupRootDir, backupId, check, sTableArray, tTableArray, isOverwrite)); } catch (Exception e){ @@ -165,9 +165,9 @@ public class RestoreDriver extends AbstractHBaseTool { return 0; } - private String getTablesForSet(String name, Configuration conf) throws IOException { - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable table = new BackupSystemTable(conn)) { + private String getTablesForSet(Connection conn, String name, Configuration conf) + throws IOException { + try (final BackupSystemTable table = new BackupSystemTable(conn)) { List tables = table.describeBackupSet(name); if (tables == null) return null; return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java index bea466b..a8d21bb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java @@ -80,13 +80,13 @@ public class BackupManager implements Closeable { * @param conf configuration * @throws IOException exception */ - public BackupManager(Configuration conf) throws IOException { + public BackupManager(Connection conn, Configuration conf) throws IOException { if (!conf.getBoolean(HConstants.BACKUP_ENABLE_KEY, HConstants.BACKUP_ENABLE_DEFAULT)) { throw new BackupException("HBase backup is not enabled. Check your " + HConstants.BACKUP_ENABLE_KEY + " setting."); } this.conf = conf; - this.conn = ConnectionFactory.createConnection(conf); + this.conn = conn; this.systemTable = new BackupSystemTable(conn); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreTablesProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreTablesProcedure.java index 4ec232b..214ad80 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreTablesProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreTablesProcedure.java @@ -139,14 +139,15 @@ public class RestoreTablesProcedure /** * Restore operation handle each backupImage in iterator + * @param conn the Connection * @param it: backupImage iterator - ascending * @param sTable: table to be restored * @param tTable: table to be restored to + * @param truncateIfExists truncate table if it exists * @throws IOException exception */ - private void restoreImages(Iterator it, TableName sTable, - TableName tTable, boolean truncateIfExists) - throws IOException { + private void restoreImages(Connection conn, Iterator it, TableName sTable, + TableName tTable, boolean truncateIfExists) throws IOException { // First image MUST be image of a FULL backup BackupImage image = it.next(); @@ -180,7 +181,7 @@ public class RestoreTablesProcedure if (manifest.getType() == BackupType.FULL || converted) { LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from " + (converted ? "converted" : "full") + " backup image " + tableBackupPath.toString()); - restoreTool.fullRestoreTable(tableBackupPath, sTable, tTable, + restoreTool.fullRestoreTable(conn, tableBackupPath, sTable, tTable, converted, truncateIfExists, lastIncrBackupId); } else { // incremental Backup throw new IOException("Unexpected backup type " + image.getType()); @@ -194,7 +195,7 @@ public class RestoreTablesProcedure String[] sarr = new String[logDirList.size()]; logDirList.toArray(sarr); Path[] paths = org.apache.hadoop.util.StringUtils.stringToPath(sarr); - restoreTool.incrementalRestoreTable(tableBackupPath, paths, new TableName[] { sTable }, + restoreTool.incrementalRestoreTable(conn, tableBackupPath, paths, new TableName[] { sTable }, new TableName[] { tTable }, lastIncrBackupId); } LOG.info(sTable + " has been successfully restored to " + tTable); @@ -202,13 +203,15 @@ public class RestoreTablesProcedure /** * Restore operation. Stage 2: resolved Backup Image dependency + * @param conn the Connection * @param backupManifestMap : tableName, Manifest * @param sTableArray The array of tables to be restored * @param tTableArray The array of mapping tables to restore to + * @param isOverwrite overwrite * @return set of BackupImages restored * @throws IOException exception */ - private void restoreStage(HashMap backupManifestMap, + private void restoreStage(Connection conn, HashMap backupManifestMap, TableName[] sTableArray, TableName[] tTableArray, boolean isOverwrite) throws IOException { TreeSet restoreImageSet = new TreeSet(); boolean truncateIfExists = isOverwrite; @@ -224,7 +227,7 @@ public class RestoreTablesProcedure list.addAll(depList); TreeSet restoreList = new TreeSet(list); LOG.debug("need to clear merged Image. to be implemented in future jira"); - restoreImages(restoreList.iterator(), table, tTableArray[i], truncateIfExists); + restoreImages(conn, restoreList.iterator(), table, tTableArray[i], truncateIfExists); restoreImageSet.addAll(restoreList); if (restoreImageSet != null && !restoreImageSet.isEmpty()) { @@ -273,7 +276,8 @@ public class RestoreTablesProcedure Path rootPath = new Path(targetRootDir); HBackupFileSystem.checkImageManifestExist(backupManifestMap, sTableArray, conf, rootPath, backupId); - restoreStage(backupManifestMap, sTableArray, tTableArray, isOverwrite); + restoreStage(env.getMasterServices().getConnection(), backupManifestMap, sTableArray, + tTableArray, isOverwrite); return Flow.NO_MORE_STATE; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java index 1862d97..f2d2615 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java @@ -96,7 +96,8 @@ public class FullTableBackupProcedure public FullTableBackupProcedure(final MasterProcedureEnv env, final String backupId, List tableList, String targetRootDir, final int workers, final long bandwidth) throws IOException { - backupManager = new BackupManager(env.getMasterConfiguration()); + backupManager = new BackupManager(env.getMasterServices().getConnection(), + env.getMasterConfiguration()); this.backupId = backupId; this.tableList = tableList; this.targetRootDir = targetRootDir; @@ -489,7 +490,8 @@ public class FullTableBackupProcedure } if (backupManager == null) { try { - backupManager = new BackupManager(env.getMasterConfiguration()); + backupManager = new BackupManager(env.getMasterServices().getConnection(), + env.getMasterConfiguration()); } catch (IOException ioe) { setFailure("full backup", ioe); return Flow.NO_MORE_STATE; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/IncrementalTableBackupProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/IncrementalTableBackupProcedure.java index db8e582..0a54e8b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/IncrementalTableBackupProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/IncrementalTableBackupProcedure.java @@ -79,7 +79,8 @@ public class IncrementalTableBackupProcedure final String backupId, List tableList, String targetRootDir, final int workers, final long bandwidth) throws IOException { - backupManager = new BackupManager(env.getMasterConfiguration()); + backupManager = new BackupManager(env.getMasterServices().getConnection(), + env.getMasterConfiguration()); this.backupId = backupId; this.tableList = tableList; this.targetRootDir = targetRootDir; @@ -203,7 +204,8 @@ public class IncrementalTableBackupProcedure } if (backupManager == null) { try { - backupManager = new BackupManager(env.getMasterConfiguration()); + backupManager = new BackupManager(env.getMasterServices().getConnection(), + env.getMasterConfiguration()); } catch (IOException ioe) { setFailure("incremental backup", ioe); } @@ -233,7 +235,8 @@ public class IncrementalTableBackupProcedure case INCREMENTAL_COPY: try { // copy out the table and region info files for each table - BackupServerUtil.copyTableRegionInfo(backupContext, conf); + BackupServerUtil.copyTableRegionInfo(env.getMasterServices().getConnection(), + backupContext, conf); incrementalCopy(backupContext); // Save list of WAL files copied backupManager.recordWALFiles(backupContext.getIncrBackupFileList()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupServerUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupServerUtil.java index 37c8d65..5e0f3c4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupServerUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupServerUtil.java @@ -150,22 +150,20 @@ public final class BackupServerUtil { /** * copy out Table RegionInfo into incremental backup image need to consider move this logic into * HBackupFileSystem + * @conn the Connection * @param backupContext backup context * @param conf configuration * @throws IOException exception * @throws InterruptedException exception */ - public static void copyTableRegionInfo(BackupInfo backupContext, Configuration conf) - throws IOException, InterruptedException { - + public static void copyTableRegionInfo(Connection conn, BackupInfo backupContext, + Configuration conf) throws IOException, InterruptedException { Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); // for each table in the table set, copy out the table info and region // info files in the correct directory structure - try (Connection conn = ConnectionFactory.createConnection(conf); - Admin admin = conn.getAdmin()) { - + try (Admin admin = conn.getAdmin()) { for (TableName table : backupContext.getTables()) { if(!admin.tableExists(table)) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreServerUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreServerUtil.java index e179a73..007ca9e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreServerUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreServerUtil.java @@ -39,6 +39,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupInfo; import org.apache.hadoop.hbase.backup.BackupRestoreServerFactory; @@ -186,8 +187,8 @@ public class RestoreServerUtil { * @param incrBackupId incremental backup Id * @throws IOException exception */ - public void incrementalRestoreTable(Path tableBackupPath, Path[] logDirs, TableName[] tableNames, - TableName[] newTableNames, String incrBackupId) throws IOException { + public void incrementalRestoreTable(Connection conn, Path tableBackupPath, Path[] logDirs, + TableName[] tableNames, TableName[] newTableNames, String incrBackupId) throws IOException { if (tableNames.length != newTableNames.length) { throw new IOException("Number of source tables and target tables does not match!"); @@ -196,10 +197,9 @@ public class RestoreServerUtil { // for incremental backup image, expect the table already created either by user or previous // full backup. Here, check that all new tables exists - try (Connection conn = ConnectionFactory.createConnection(conf); - Admin admin = conn.getAdmin()) { + try (Admin admin = conn.getAdmin()) { for (TableName tableName : newTableNames) { - if (!admin.tableExists(tableName)) { + if (!MetaTableAccessor.tableExists(conn, tableName)) { admin.close(); throw new IOException("HBase table " + tableName + " does not exist. Create the table first, e.g. by restoring a full backup."); @@ -241,9 +241,10 @@ public class RestoreServerUtil { } } - public void fullRestoreTable(Path tableBackupPath, TableName tableName, TableName newTableName, - boolean converted, boolean truncateIfExists, String lastIncrBackupId) throws IOException { - restoreTableAndCreate(tableName, newTableName, tableBackupPath, converted, truncateIfExists, + public void fullRestoreTable(Connection conn, Path tableBackupPath, TableName tableName, + TableName newTableName, boolean converted, boolean truncateIfExists, String lastIncrBackupId) + throws IOException { + restoreTableAndCreate(conn, tableName, newTableName, tableBackupPath, converted, truncateIfExists, lastIncrBackupId); } @@ -362,7 +363,7 @@ public class RestoreServerUtil { return null; } - private void restoreTableAndCreate(TableName tableName, TableName newTableName, + private void restoreTableAndCreate(Connection conn, TableName tableName, TableName newTableName, Path tableBackupPath, boolean converted, boolean truncateIfExists, String lastIncrBackupId) throws IOException { if (newTableName == null || newTableName.equals("")) { @@ -376,8 +377,7 @@ public class RestoreServerUtil { LOG.debug("Retrieved descriptor: " + tableDescriptor + " thru " + lastIncrBackupId); } - try (Connection conn = ConnectionFactory.createConnection(conf); - HBaseAdmin hbadmin = (HBaseAdmin) conn.getAdmin();) { + try (HBaseAdmin hbadmin = (HBaseAdmin) conn.getAdmin();) { if (tableDescriptor == null) { Path tableSnapshotPath = getTableSnapshotPath(backupRootPath, tableName, backupId); if (fileSys.exists(tableSnapshotPath)) {