diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 5b5dccd..c4996f9 100644
--- hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -1803,6 +1803,26 @@ public interface Admin extends Abortable, Closeable {
*/
void restoreSnapshot(byte[] snapshotName) throws IOException, RestoreSnapshotException;
+ /**
+ * Restore the specified snapshot on the original table. (The table must be disabled) If
+ * 'takeFailSafeSnapshot' is set to true, a snapshot of the current table is taken before
+ * executing the restore operation. In case of restore failure, the failsafe snapshot will be
+ * restored. If the restore completes without problem the failsafe snapshot is deleted. The
+ * failsafe snapshot name is configurable by using the property
+ * "hbase.snapshot.restore.failsafe.name".
+ * @param snapshotName name of the snapshot to restore
+ * @param takeFailSafeSnapshot true if the failsafe snapshot should be taken
+ * @param restoreAcl true to restore acl of snapshot
+ * @param moveMobFilesFromArchiveToWorkingDir true to move mob files from archive to working
+ * directory
+ * @throws IOException if a remote or network exception occurs
+ * @throws RestoreSnapshotException if snapshot failed to be restored
+ * @throws IllegalArgumentException if the restore request is formatted incorrectly
+ */
+ void restoreSnapshot(final String snapshotName, final boolean takeFailSafeSnapshot,
+ final boolean restoreAcl, final boolean moveMobFilesFromArchiveToWorkingDir)
+ throws IOException, RestoreSnapshotException;
+
/**
* Restore the specified snapshot on the original table. (The table must be disabled) If the
* "hbase.snapshot.restore.take.failsafe.snapshot" configuration property is set to true, a
diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index e4bb675..c818d49 100644
--- hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -2662,11 +2662,19 @@ public class HBaseAdmin implements Admin {
@Override
public void restoreSnapshot(final String snapshotName, final boolean takeFailSafeSnapshot,
final boolean restoreAcl) throws IOException, RestoreSnapshotException {
+ restoreSnapshot(snapshotName, takeFailSafeSnapshot, restoreAcl, false);
+ }
+
+ @Override
+ public void restoreSnapshot(final String snapshotName, final boolean takeFailSafeSnapshot,
+ final boolean restoreAcl, final boolean moveMobFilesToWorkingDir)
+ throws IOException, RestoreSnapshotException {
TableName tableName = getTableNameBeforeRestoreSnapshot(snapshotName);
// The table does not exists, switch to clone.
if (!tableExists(tableName)) {
- cloneSnapshot(snapshotName, tableName, restoreAcl);
+ get(internalRestoreSnapshotAsync(snapshotName, tableName, restoreAcl,
+ moveMobFilesToWorkingDir), Integer.MAX_VALUE, TimeUnit.MILLISECONDS);
return;
}
@@ -2691,7 +2699,7 @@ public class HBaseAdmin implements Admin {
try {
// Restore snapshot
get(
- internalRestoreSnapshotAsync(snapshotName, tableName, restoreAcl),
+ internalRestoreSnapshotAsync(snapshotName, tableName, restoreAcl, moveMobFilesToWorkingDir),
syncWaitTimeout,
TimeUnit.MILLISECONDS);
} catch (IOException e) {
@@ -2700,7 +2708,7 @@ public class HBaseAdmin implements Admin {
if (takeFailSafeSnapshot) {
try {
get(
- internalRestoreSnapshotAsync(failSafeSnapshotSnapshotName, tableName, restoreAcl),
+ internalRestoreSnapshotAsync(failSafeSnapshotSnapshotName, tableName, restoreAcl, false),
syncWaitTimeout,
TimeUnit.MILLISECONDS);
String msg = "Restore snapshot=" + snapshotName +
@@ -2743,7 +2751,7 @@ public class HBaseAdmin implements Admin {
throw new TableNotDisabledException(tableName);
}
- return internalRestoreSnapshotAsync(snapshotName, tableName, false);
+ return internalRestoreSnapshotAsync(snapshotName, tableName, false, false);
}
@Override
@@ -2759,7 +2767,7 @@ public class HBaseAdmin implements Admin {
throw new TableExistsException(tableName);
}
get(
- internalRestoreSnapshotAsync(snapshotName, tableName, restoreAcl),
+ internalRestoreSnapshotAsync(snapshotName, tableName, restoreAcl, false),
Integer.MAX_VALUE,
TimeUnit.MILLISECONDS);
}
@@ -2776,7 +2784,7 @@ public class HBaseAdmin implements Admin {
if (tableExists(tableName)) {
throw new TableExistsException(tableName);
}
- return internalRestoreSnapshotAsync(snapshotName, tableName, false);
+ return internalRestoreSnapshotAsync(snapshotName, tableName, false, false);
}
@Override
@@ -2860,12 +2868,16 @@ public class HBaseAdmin implements Admin {
* create an HTable instance to this table before it is available.
* @param snapshotName snapshot to restore
* @param tableName table name to restore the snapshot on
+ * @param restoreAcl boolean flag to tell if ACLs need to be restored
+ * @param moveMobFilesFromArchiveToWorkDir boolean flag to tell if MOB files need to be moved from
+ * archive to table directory.
* @throws IOException if a remote or network exception occurs
* @throws RestoreSnapshotException if snapshot failed to be restored
* @throws IllegalArgumentException if the restore request is formatted incorrectly
*/
private Future internalRestoreSnapshotAsync(final String snapshotName,
- final TableName tableName, final boolean restoreAcl)
+ final TableName tableName, final boolean restoreAcl,
+ final boolean moveMobFilesFromArchiveToWorkDir)
throws IOException, RestoreSnapshotException {
final SnapshotProtos.SnapshotDescription snapshot = SnapshotProtos.SnapshotDescription.newBuilder()
.setName(snapshotName).setTable(tableName.getNameAsString()).build();
@@ -2882,6 +2894,7 @@ public class HBaseAdmin implements Admin {
.setNonceGroup(ng.getNonceGroup())
.setNonce(ng.newNonce())
.setRestoreACL(restoreAcl)
+ .setMoveMobFilesFromArchiveToWorkDir(moveMobFilesFromArchiveToWorkDir)
.build();
return master.restoreSnapshot(getRpcController(), request);
}
diff --git hbase-protocol-shaded/src/main/protobuf/Master.proto hbase-protocol-shaded/src/main/protobuf/Master.proto
index 7cdd143..b7106de 100644
--- hbase-protocol-shaded/src/main/protobuf/Master.proto
+++ hbase-protocol-shaded/src/main/protobuf/Master.proto
@@ -421,6 +421,7 @@ message RestoreSnapshotRequest {
optional uint64 nonce_group = 2 [default = 0];
optional uint64 nonce = 3 [default = 0];
optional bool restoreACL = 4 [default = false];
+ optional bool moveMobFilesFromArchiveToWorkDir = 5 [default = false];
}
message RestoreSnapshotResponse {
diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
index 4da1235..f5527fa 100644
--- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
@@ -215,6 +215,47 @@ public class HFileArchiver {
}
}
+ /**
+ * Move Store File from Archive to family directory
+ * @param conf {@link Configuration} to examine to determine the archive directory
+ * @param fs the filesystem where the store files live
+ * @param regionInfo region hosting the store files
+ * @param tableDir {@link Path} to where the table is being stored (for building the archive path)
+ * @param familyDir the family hosting the store files
+ * @param storeFile file to be moved
+ * @throws IOException if the files could not be correctly disposed.
+ */
+ public static void moveStoreFileFromArchiveToFamilyDir(Configuration conf, FileSystem fs,
+ RegionInfo regionInfo, Path tableDir, Path familyDir, String storeFile) throws IOException {
+ Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir,
+ familyDir.getName().getBytes());
+
+ // The caller makes sure that Dst File does not exist and Src File exists
+
+ File archiveFile = new FileablePath(fs, new Path(storeArchiveDir, storeFile));
+
+ // Make sure the destination directory exists
+ if (!fs.exists(familyDir)) {
+ if (!fs.mkdirs(familyDir)) {
+ throw new IOException("Failed to create the family directory:" + familyDir);
+ }
+ }
+
+ Boolean success = false;
+ try {
+ success = archiveFile.moveAndClose(new Path(familyDir, storeFile));
+ } catch (FileNotFoundException fnfe) {
+ LOG.warn("Failed to move " + archiveFile +
+ " because it does not exist! Skipping and continuing on.", fnfe);
+ } catch (IOException e) {
+ LOG.warn("Failed to move " + archiveFile + " ", e);
+ }
+
+ if (!success) {
+ LOG.error("Failed to move " + archiveFile + " to " + familyDir);
+ }
+ }
+
/**
* Remove the store files, either by archiving them or outright deletion
* @param conf {@link Configuration} to examine to determine the archive directory
diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index a34f3f4..51a07db 100644
--- hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -2387,6 +2387,13 @@ public class HMaster extends HRegionServer implements MasterServices {
public long restoreSnapshot(final SnapshotDescription snapshotDesc,
final long nonceGroup, final long nonce, final boolean restoreAcl) throws IOException {
+ return restoreSnapshot(snapshotDesc, nonceGroup, nonce, restoreAcl, false);
+ }
+
+ public long restoreSnapshot(final SnapshotDescription snapshotDesc,
+ final long nonceGroup, final long nonce, final boolean restoreAcl,
+ final boolean moveMobFilesFromArchiveToWorkDir)
+ throws IOException {
checkInitialized();
getSnapshotManager().checkSnapshotSupport();
@@ -2399,7 +2406,8 @@ public class HMaster extends HRegionServer implements MasterServices {
@Override
protected void run() throws IOException {
setProcId(
- getSnapshotManager().restoreOrCloneSnapshot(snapshotDesc, getNonceKey(), restoreAcl));
+ getSnapshotManager().restoreOrCloneSnapshot(snapshotDesc, getNonceKey(), restoreAcl,
+ moveMobFilesFromArchiveToWorkDir));
}
@Override
diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 1bd6487..41daf27 100644
--- hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -1382,7 +1382,7 @@ public class MasterRpcServices extends RSRpcServices
RestoreSnapshotRequest request) throws ServiceException {
try {
long procId = master.restoreSnapshot(request.getSnapshot(), request.getNonceGroup(),
- request.getNonce(), request.getRestoreACL());
+ request.getNonce(), request.getRestoreACL(), request.getMoveMobFilesFromArchiveToWorkDir());
return RestoreSnapshotResponse.newBuilder().setProcId(procId).build();
} catch (ForeignException e) {
throw new ServiceException(e.getCause());
diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
index 6155f16..d8666fc 100644
--- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
@@ -71,6 +71,7 @@ public class CloneSnapshotProcedure
private SnapshotDescription snapshot;
private boolean restoreAcl;
private List newRegions = null;
+ private boolean moveMobFilesFromArchiveToWorkDir;
private Map > parentsToChildrenPairMap = new HashMap<>();
// Monitor
@@ -106,6 +107,18 @@ public class CloneSnapshotProcedure
getMonitorStatus();
}
+ public CloneSnapshotProcedure(final MasterProcedureEnv env,
+ final TableDescriptor tableDescriptor, final SnapshotDescription snapshot,
+ final boolean restoreAcl, final boolean moveMobFilesFromArchiveToWorkDir) {
+ super(env);
+ this.tableDescriptor = tableDescriptor;
+ this.snapshot = snapshot;
+ this.restoreAcl = restoreAcl;
+ this.moveMobFilesFromArchiveToWorkDir = moveMobFilesFromArchiveToWorkDir;
+
+ getMonitorStatus();
+ }
+
/**
* Set up monitor status if it is not created.
*/
@@ -393,7 +406,8 @@ public class CloneSnapshotProcedure
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot);
RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(
conf, fs, manifest, tableDescriptor, tableRootDir, monitorException, monitorStatus);
- RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
+ RestoreSnapshotHelper.RestoreMetaChanges metaChanges =
+ restoreHelper.restoreHdfsRegions(moveMobFilesFromArchiveToWorkDir);
// Clone operation should not have stuff to restore or remove
Preconditions.checkArgument(
diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
index 2cf5584..7a11c7c 100644
--- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
@@ -69,6 +69,7 @@ public class RestoreSnapshotProcedure
private SnapshotDescription snapshot;
private boolean restoreAcl;
+ private boolean moveMobFilesFromArchiveToWorkDir;
// Monitor
private MonitoredTask monitorStatus = null;
@@ -83,7 +84,7 @@ public class RestoreSnapshotProcedure
public RestoreSnapshotProcedure(final MasterProcedureEnv env,
final TableDescriptor tableDescriptor, final SnapshotDescription snapshot) {
- this(env, tableDescriptor, snapshot, false);
+ this(env, tableDescriptor, snapshot, false, false);
}
/**
* Constructor
@@ -96,13 +97,15 @@ public class RestoreSnapshotProcedure
final MasterProcedureEnv env,
final TableDescriptor tableDescriptor,
final SnapshotDescription snapshot,
- final boolean restoreAcl) {
+ final boolean restoreAcl,
+ final boolean moveMobFileFromArchiveToWorkDir) {
super(env);
// This is the new schema we are going to write out as this modification.
this.modifiedTableDescriptor = tableDescriptor;
// Snapshot information
this.snapshot = snapshot;
this.restoreAcl = restoreAcl;
+ this.moveMobFilesFromArchiveToWorkDir = moveMobFileFromArchiveToWorkDir;
// Monitor
getMonitorStatus();
@@ -141,7 +144,7 @@ public class RestoreSnapshotProcedure
setNextState(RestoreSnapshotState.RESTORE_SNAPSHOT_WRITE_FS_LAYOUT);
break;
case RESTORE_SNAPSHOT_WRITE_FS_LAYOUT:
- restoreSnapshot(env);
+ restoreSnapshot(env, moveMobFilesFromArchiveToWorkDir);
setNextState(RestoreSnapshotState.RESTORE_SNAPSHOT_UPDATE_META);
break;
case RESTORE_SNAPSHOT_UPDATE_META:
@@ -373,7 +376,8 @@ public class RestoreSnapshotProcedure
* @param env MasterProcedureEnv
* @throws IOException
**/
- private void restoreSnapshot(final MasterProcedureEnv env) throws IOException {
+ private void restoreSnapshot(final MasterProcedureEnv env,
+ final boolean moveMobFilesFromArchiveToWorkDir) throws IOException {
MasterFileSystem fileSystemManager = env.getMasterServices().getMasterFileSystem();
FileSystem fs = fileSystemManager.getFileSystem();
Path rootDir = fileSystemManager.getRootDir();
@@ -393,7 +397,8 @@ public class RestoreSnapshotProcedure
monitorException,
getMonitorStatus());
- RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
+ RestoreSnapshotHelper.RestoreMetaChanges metaChanges =
+ restoreHelper.restoreHdfsRegions(moveMobFilesFromArchiveToWorkDir);
regionsToRestore = metaChanges.getRegionsToRestore();
regionsToRemove = metaChanges.getRegionsToRemove();
regionsToAdd = metaChanges.getRegionsToAdd();
diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index 20a4f39..b4f6230 100644
--- hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -692,7 +692,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
*/
private long cloneSnapshot(final SnapshotDescription reqSnapshot, final TableName tableName,
final SnapshotDescription snapshot, final TableDescriptor snapshotTableDesc,
- final NonceKey nonceKey, final boolean restoreAcl) throws IOException {
+ final NonceKey nonceKey, final boolean restoreAcl, final boolean moveMobFilesToWorkDir)
+ throws IOException {
MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
TableDescriptor htd = TableDescriptorBuilder.copy(tableName, snapshotTableDesc);
org.apache.hadoop.hbase.client.SnapshotDescription snapshotPOJO = null;
@@ -702,7 +703,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
}
long procId;
try {
- procId = cloneSnapshot(snapshot, htd, nonceKey, restoreAcl);
+ procId = cloneSnapshot(snapshot, htd, nonceKey, restoreAcl, moveMobFilesToWorkDir);
} catch (IOException e) {
LOG.error("Exception occurred while cloning the snapshot " + snapshot.getName()
+ " as table " + tableName.getNameAsString(), e);
@@ -726,7 +727,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
* @return procId the ID of the clone snapshot procedure
*/
synchronized long cloneSnapshot(final SnapshotDescription snapshot,
- final TableDescriptor tableDescriptor, final NonceKey nonceKey, final boolean restoreAcl)
+ final TableDescriptor tableDescriptor, final NonceKey nonceKey, final boolean restoreAcl,
+ final boolean moveMobFilesToWorkDir)
throws HBaseSnapshotException {
TableName tableName = tableDescriptor.getTableName();
@@ -743,7 +745,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
try {
long procId = master.getMasterProcedureExecutor().submitProcedure(
new CloneSnapshotProcedure(master.getMasterProcedureExecutor().getEnvironment(),
- tableDescriptor, snapshot, restoreAcl),
+ tableDescriptor, snapshot, restoreAcl, moveMobFilesToWorkDir),
nonceKey);
this.restoreTableToProcIdMap.put(tableName, procId);
return procId;
@@ -763,6 +765,11 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
*/
public long restoreOrCloneSnapshot(final SnapshotDescription reqSnapshot, final NonceKey nonceKey,
final boolean restoreAcl) throws IOException {
+ return restoreOrCloneSnapshot(reqSnapshot, nonceKey, restoreAcl, false);
+ }
+
+ public long restoreOrCloneSnapshot(final SnapshotDescription reqSnapshot, final NonceKey nonceKey,
+ final boolean restoreAcl, final boolean moveMobFiles) throws IOException {
FileSystem fs = master.getMasterFileSystem().getFileSystem();
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
@@ -792,10 +799,11 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
long procId;
if (MetaTableAccessor.tableExists(master.getConnection(), tableName)) {
procId = restoreSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey,
- restoreAcl);
+ restoreAcl, moveMobFiles);
} else {
procId =
- cloneSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey, restoreAcl);
+ cloneSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey,
+ restoreAcl, moveMobFiles);
}
return procId;
}
@@ -814,7 +822,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
*/
private long restoreSnapshot(final SnapshotDescription reqSnapshot, final TableName tableName,
final SnapshotDescription snapshot, final TableDescriptor snapshotTableDesc,
- final NonceKey nonceKey, final boolean restoreAcl) throws IOException {
+ final NonceKey nonceKey, final boolean restoreAcl, final boolean moveMobFiles) throws IOException {
MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
if (master.getTableStateManager().isTableState(
@@ -833,7 +841,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
long procId;
try {
- procId = restoreSnapshot(snapshot, snapshotTableDesc, nonceKey, restoreAcl);
+ procId = restoreSnapshot(snapshot, snapshotTableDesc, nonceKey, restoreAcl, moveMobFiles);
} catch (IOException e) {
LOG.error("Exception occurred while restoring the snapshot " + snapshot.getName()
+ " as table " + tableName.getNameAsString(), e);
@@ -858,7 +866,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
* @return procId the ID of the restore snapshot procedure
*/
private synchronized long restoreSnapshot(final SnapshotDescription snapshot,
- final TableDescriptor tableDescriptor, final NonceKey nonceKey, final boolean restoreAcl)
+ final TableDescriptor tableDescriptor, final NonceKey nonceKey, final boolean restoreAcl,
+ final boolean moveMobFiles)
throws HBaseSnapshotException {
final TableName tableName = tableDescriptor.getTableName();
@@ -875,7 +884,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
try {
long procId = master.getMasterProcedureExecutor().submitProcedure(
new RestoreSnapshotProcedure(master.getMasterProcedureExecutor().getEnvironment(),
- tableDescriptor, snapshot, restoreAcl),
+ tableDescriptor, snapshot, restoreAcl, moveMobFiles),
nonceKey);
this.restoreTableToProcIdMap.put(tableName, procId);
return procId;
diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
index e08d547..c851e55 100644
--- hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -175,13 +175,29 @@ public class RestoreSnapshotHelper {
public RestoreMetaChanges restoreHdfsRegions() throws IOException {
ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "RestoreSnapshot");
try {
- return restoreHdfsRegions(exec);
+ return restoreHdfsRegions(exec, false);
} finally {
exec.shutdown();
}
}
- private RestoreMetaChanges restoreHdfsRegions(final ThreadPoolExecutor exec) throws IOException {
+ /**
+ * Restore the on-disk table to a specified snapshot state.
+ * @param moveMobFilesFromArchiveToWorkDir Whether move mob files from archive to working dir
+ * @return the set of regions touched by the restore operation
+ */
+ public RestoreMetaChanges restoreHdfsRegions(final boolean moveMobFilesFromArchiveToWorkDir)
+ throws IOException {
+ ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "RestoreSnapshot");
+ try {
+ return restoreHdfsRegions(exec, moveMobFilesFromArchiveToWorkDir);
+ } finally {
+ exec.shutdown();
+ }
+ }
+
+ private RestoreMetaChanges restoreHdfsRegions(final ThreadPoolExecutor exec,
+ final boolean moveMobFilesFromArchiveToWorkDir) throws IOException {
LOG.info("starting restore table regions using snapshot=" + snapshotDesc);
Map regionManifests = snapshotManifest.getRegionManifestsMap();
@@ -222,7 +238,7 @@ public class RestoreSnapshotHelper {
// restore the mob region in case
List mobRegions = new ArrayList<>(1);
mobRegions.add(mobRegion);
- restoreHdfsMobRegions(exec, regionManifests, mobRegions);
+ restoreHdfsMobRegions(exec, regionManifests, mobRegions, moveMobFilesFromArchiveToWorkDir);
regionNames.remove(mobRegion.getEncodedName());
}
restoreHdfsRegions(exec, regionManifests, metaChanges.getRegionsToRestore());
@@ -242,7 +258,7 @@ public class RestoreSnapshotHelper {
monitor.rethrowException();
// add the mob region
if (regionNames.contains(mobRegion.getEncodedName())) {
- cloneHdfsMobRegion(regionManifests, mobRegion);
+ cloneHdfsMobRegion(regionManifests, mobRegion, moveMobFilesFromArchiveToWorkDir);
regionNames.remove(mobRegion.getEncodedName());
}
for (String regionName: regionNames) {
@@ -435,12 +451,14 @@ public class RestoreSnapshotHelper {
*/
private void restoreHdfsMobRegions(final ThreadPoolExecutor exec,
final Map regionManifests,
- final List regions) throws IOException {
+ final List regions, final boolean moveMobFilesFromArchiveToWorkDir)
+ throws IOException {
if (regions == null || regions.isEmpty()) return;
ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
@Override
public void editRegion(final RegionInfo hri) throws IOException {
- restoreMobRegion(hri, regionManifests.get(hri.getEncodedName()));
+ restoreMobRegion(hri, regionManifests.get(hri.getEncodedName()),
+ moveMobFilesFromArchiveToWorkDir);
}
});
}
@@ -462,7 +480,8 @@ public class RestoreSnapshotHelper {
*/
private void restoreRegion(final RegionInfo regionInfo,
final SnapshotRegionManifest regionManifest) throws IOException {
- restoreRegion(regionInfo, regionManifest, new Path(tableDir, regionInfo.getEncodedName()));
+ restoreRegion(regionInfo, regionManifest, new Path(tableDir, regionInfo.getEncodedName()),
+ false);
}
/**
@@ -470,12 +489,13 @@ public class RestoreSnapshotHelper {
* and adding the missing ones from the snapshot.
*/
private void restoreMobRegion(final RegionInfo regionInfo,
- final SnapshotRegionManifest regionManifest) throws IOException {
+ final SnapshotRegionManifest regionManifest,
+ final boolean moveMobFilesFromArchiveToWorkDir) throws IOException {
if (regionManifest == null) {
return;
}
restoreRegion(regionInfo, regionManifest,
- MobUtils.getMobRegionPath(conf, tableDesc.getTableName()));
+ MobUtils.getMobRegionPath(conf, tableDesc.getTableName()), moveMobFilesFromArchiveToWorkDir);
}
/**
@@ -483,7 +503,8 @@ public class RestoreSnapshotHelper {
* and adding the missing ones from the snapshot.
*/
private void restoreRegion(final RegionInfo regionInfo,
- final SnapshotRegionManifest regionManifest, Path regionDir) throws IOException {
+ final SnapshotRegionManifest regionManifest, Path regionDir,
+ final boolean moveHFilesFromArchiveToFamilyDir) throws IOException {
Map> snapshotFiles =
getRegionHFileReferences(regionManifest);
@@ -519,7 +540,8 @@ public class RestoreSnapshotHelper {
for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) {
LOG.debug("Adding HFileLink " + storeFile.getName() +
" to region=" + regionInfo.getEncodedName() + " table=" + tableName);
- restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
+ restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs,
+ moveHFilesFromArchiveToFamilyDir);
}
} else {
// Family doesn't exists in the snapshot
@@ -540,7 +562,8 @@ public class RestoreSnapshotHelper {
for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) {
LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
- restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
+ restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs,
+ moveHFilesFromArchiveToFamilyDir);
}
}
}
@@ -607,10 +630,11 @@ public class RestoreSnapshotHelper {
* and create a HFileLink for each hfile.
*/
private void cloneHdfsMobRegion(final Map regionManifests,
- final RegionInfo region) throws IOException {
+ final RegionInfo region, final boolean moveMobFilesFromArchiveToWorkDir) throws IOException {
// clone region info (change embedded tableName with the new one)
Path clonedRegionPath = MobUtils.getMobRegionPath(conf, tableDesc.getTableName());
- cloneRegion(clonedRegionPath, region, regionManifests.get(region.getEncodedName()));
+ cloneRegion(clonedRegionPath, region, regionManifests.get(region.getEncodedName()),
+ moveMobFilesFromArchiveToWorkDir);
}
/**
@@ -625,13 +649,15 @@ public class RestoreSnapshotHelper {
* @param snapshotRegionInfo
*/
private void cloneRegion(final Path regionDir, final RegionInfo snapshotRegionInfo,
- final SnapshotRegionManifest manifest) throws IOException {
+ final SnapshotRegionManifest manifest, final boolean moveMobFilesFromArchiveToWorkDir)
+ throws IOException {
final String tableName = tableDesc.getTableName().getNameAsString();
for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
- restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
+ restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs,
+ moveMobFilesFromArchiveToWorkDir);
}
}
}
@@ -650,7 +676,7 @@ public class RestoreSnapshotHelper {
private void cloneRegion(final HRegion region, final RegionInfo snapshotRegionInfo,
final SnapshotRegionManifest manifest) throws IOException {
cloneRegion(new Path(tableDir, region.getRegionInfo().getEncodedName()), snapshotRegionInfo,
- manifest);
+ manifest, false);
}
/**
@@ -667,7 +693,8 @@ public class RestoreSnapshotHelper {
* @param storeFile store file name (can be a Reference, HFileLink or simple HFile)
*/
private void restoreStoreFile(final Path familyDir, final RegionInfo regionInfo,
- final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef)
+ final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef,
+ final boolean moveHFilesFromArchiveToFamilyDir)
throws IOException {
String hfileName = storeFile.getName();
if (HFileLink.isHFileLink(hfileName)) {
@@ -675,7 +702,13 @@ public class RestoreSnapshotHelper {
} else if (StoreFileInfo.isReference(hfileName)) {
restoreReferenceFile(familyDir, regionInfo, storeFile);
} else {
- HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef);
+ if (moveHFilesFromArchiveToFamilyDir) {
+ // Move hfiles from archive to family directory
+ HFileArchiver.moveStoreFileFromArchiveToFamilyDir(conf, fs, regionInfo,
+ tableDir, familyDir, hfileName);
+ } else {
+ HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef);
+ }
}
}
diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobRestoreSnapshotFromClient.java hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobRestoreSnapshotFromClient.java
index bde6f0f..ebb995a 100644
--- hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobRestoreSnapshotFromClient.java
+++ hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobRestoreSnapshotFromClient.java
@@ -19,15 +19,25 @@ package org.apache.hadoop.hbase.client;
import java.io.IOException;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
import org.junit.BeforeClass;
+import org.junit.Test;
import org.junit.experimental.categories.Category;
/**
@@ -35,11 +45,13 @@ import org.junit.experimental.categories.Category;
*/
@Category({ClientTests.class, LargeTests.class})
public class TestMobRestoreSnapshotFromClient extends TestRestoreSnapshotFromClient {
+ private static FileSystem fs;
@BeforeClass
public static void setupCluster() throws Exception {
setupConf(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
+ fs = TEST_UTIL.getMiniHBaseCluster().getMaster().getFileSystem();
}
protected static void setupConf(Configuration conf) {
@@ -70,4 +82,111 @@ public class TestMobRestoreSnapshotFromClient extends TestRestoreSnapshotFromCli
protected int countRows(final Table table, final byte[]... families) throws IOException {
return MobSnapshotTestingUtils.countMobRows(table, families);
}
+
+ private void restoreSnapshot(final boolean moveMobFiles) throws IOException {
+
+ Path mobDir = new Path(new Path(
+ new Path(TEST_UTIL.getDefaultRootDirPath(), MobConstants.MOB_DIR_NAME), "data"), "default");
+ FileStatus[] r1 = fs.listStatus(mobDir);
+ assertTrue(r1.length >= 1);
+
+ admin.disableTable(tableName);
+
+ admin.snapshot(snapshotName1, tableName);
+ admin.deleteTable(tableName);
+
+ // Restore from snapshot-0
+ admin.restoreSnapshot(Bytes.toString(snapshotName0), false, false, moveMobFiles);
+
+ verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
+ SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
+
+ FileStatus[] r = fs.listStatus(new Path(mobDir, tableName.getNameAsString()));
+
+ r = fs.listStatus(r[0].getPath());
+ r = fs.listStatus(r[0].getPath());
+
+ assertTrue(r.length > 0);
+
+ if (moveMobFiles) {
+ // For each of the mob file, verify that it is not a HFileLink
+ for (FileStatus rr : r) {
+ assertFalse(HFileLink.isHFileLink(rr.getPath()));
+ }
+ } else {
+ // For each of the mob file, verify that it is a HFileLink
+ for (FileStatus rr : r) {
+ assertTrue(HFileLink.isHFileLink(rr.getPath()));
+ }
+ }
+
+ // Make sure the snapshot is not corrupted after moving mob files from archive to working
+ // directory.
+ TableName clonedName = TableName.valueOf(tableName.getNameAsString() + "-c");
+ admin.cloneSnapshot(snapshotName0, clonedName);
+ verifyRowCount(TEST_UTIL, clonedName, snapshot0Rows);
+ SnapshotTestingUtils.verifyReplicasCameOnline(clonedName, admin, getNumReplicas());
+ }
+
+
+ /**
+ * Test the case with moving mob files from archive to working directory.
+ */
+ @Test
+ public void testRestoreSnapshotWithMovingMobFilesFromArchiveToWorkDir() throws IOException {
+ restoreSnapshot(true);
+ }
+
+ /**
+ * Test the case with moving mob files from archive to working directory.
+ */
+ @Test
+ public void testRestoreSnapshotCombined() throws IOException {
+ restoreSnapshot(false);
+ // disable table.
+ admin.disableTable(tableName);
+
+ // Restore from snapshot-1
+ admin.restoreSnapshot(Bytes.toString(snapshotName1), false, false, true);
+
+ // enable table and insert data
+ admin.enableTable(tableName);
+
+ verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
+ SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
+
+ Path mobDir = new Path(new Path(
+ new Path(TEST_UTIL.getDefaultRootDirPath(), MobConstants.MOB_DIR_NAME), "data"), "default");
+
+ FileStatus[] r = fs.listStatus(new Path(mobDir, tableName.getNameAsString()));
+
+ r = fs.listStatus(r[0].getPath());
+ r = fs.listStatus(r[0].getPath());
+
+ assertTrue(r.length > 0);
+
+ for (FileStatus rr : r) {
+ assertFalse(HFileLink.isHFileLink(rr.getPath()));
+ }
+
+ // Make sure the snapshot is not corrupted after moving mob files from archive to working
+ // directory.
+ TableName clonedName = TableName.valueOf(tableName.getNameAsString() + "-c1");
+ admin.cloneSnapshot(snapshotName1, clonedName);
+ verifyRowCount(TEST_UTIL, clonedName, snapshot1Rows);
+ SnapshotTestingUtils.verifyReplicasCameOnline(clonedName, admin, getNumReplicas());
+ }
+
+ /**
+ * Test the case with the original behavior.
+ */
+ @Test
+ public void testRestoreSnapshotNormal() throws IOException {
+ try {
+ restoreSnapshot(false);
+ } finally {
+
+ }
+ }
+
}
diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
index 3190fb9..2136285 100644
--- hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
+++ hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
@@ -68,13 +68,13 @@ public class TestRestoreSnapshotFromClient {
protected final byte[] TEST_FAMILY2 = Bytes.toBytes("cf2");
protected TableName tableName;
- private byte[] emptySnapshot;
- private byte[] snapshotName0;
- private byte[] snapshotName1;
- private byte[] snapshotName2;
- private int snapshot0Rows;
- private int snapshot1Rows;
- private Admin admin;
+ protected byte[] emptySnapshot;
+ protected byte[] snapshotName0;
+ protected byte[] snapshotName1;
+ protected byte[] snapshotName2;
+ protected int snapshot0Rows;
+ protected int snapshot1Rows;
+ protected Admin admin;
@Rule
public TestName name = new TestName();
@@ -91,8 +91,7 @@ public class TestRestoreSnapshotFromClient {
TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
- TEST_UTIL.getConfiguration().setBoolean(
- "hbase.master.enabletable.roundrobin", true);
+ TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
}
@AfterClass
diff --git hbase-shell/src/main/ruby/hbase/admin.rb hbase-shell/src/main/ruby/hbase/admin.rb
index 581ccb3..b6b016b 100644
--- hbase-shell/src/main/ruby/hbase/admin.rb
+++ hbase-shell/src/main/ruby/hbase/admin.rb
@@ -985,10 +985,10 @@ module Hbase
#----------------------------------------------------------------------------------------------
# Restore specified snapshot
- def restore_snapshot(snapshot_name, restore_acl = false)
+ def restore_snapshot(snapshot_name, restore_acl = false, move_mobfiles = false)
conf = @connection.getConfiguration
take_fail_safe_snapshot = conf.getBoolean('hbase.snapshot.restore.take.failsafe.snapshot', false)
- @admin.restoreSnapshot(snapshot_name, take_fail_safe_snapshot, restore_acl)
+ @admin.restoreSnapshot(snapshot_name, take_fail_safe_snapshot, restore_acl, move_mobfiles)
end
#----------------------------------------------------------------------------------------------
diff --git hbase-shell/src/main/ruby/hbase_constants.rb hbase-shell/src/main/ruby/hbase_constants.rb
index ebaae78..12df9ff 100644
--- hbase-shell/src/main/ruby/hbase_constants.rb
+++ hbase-shell/src/main/ruby/hbase_constants.rb
@@ -84,6 +84,7 @@ module HBaseConstants
SERVER_NAME = 'SERVER_NAME'.freeze
LOCALITY_THRESHOLD = 'LOCALITY_THRESHOLD'.freeze
RESTORE_ACL = 'RESTORE_ACL'.freeze
+ MOVE_MOB_FILES_FROM_ARCHIVE_TO_WORKDIR = 'MOVE_MOB_FILES_FROM_ARCHIVE_TO_WORKDIR'.freeze
FORMATTER = 'FORMATTER'.freeze
FORMATTER_CLASS = 'FORMATTER_CLASS'.freeze
POLICY = 'POLICY'.freeze
diff --git hbase-shell/src/main/ruby/shell/commands/restore_snapshot.rb hbase-shell/src/main/ruby/shell/commands/restore_snapshot.rb
index be6ee3c..d8ccd80 100644
--- hbase-shell/src/main/ruby/shell/commands/restore_snapshot.rb
+++ hbase-shell/src/main/ruby/shell/commands/restore_snapshot.rb
@@ -32,13 +32,20 @@ Examples:
Following command will restore all acl from snapshot table into the table.
hbase> restore_snapshot 'snapshotName', {RESTORE_ACL=>true}
+
+Following command will restore all mob files from archive directory to the work directory, if the
+ table is MOB enabled.
+
+ hbase> restore_snapshot 'snapshotName', {MOVE_MOB_FILES_FROM_ARCHIVE_TO_WORKDIR=>true}
+
EOF
end
def command(snapshot_name, args = {})
raise(ArgumentError, 'Arguments should be a Hash') unless args.is_a?(Hash)
restore_acl = args.delete(RESTORE_ACL) || false
- admin.restore_snapshot(snapshot_name, restore_acl)
+ move_mobfiles = args.delete(MOVE_MOB_FILES_FROM_ARCHIVE_TO_WORKDIR) || false
+ admin.restore_snapshot(snapshot_name, restore_acl, move_mobfiles)
end
end
end