diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java index e4a3b58..d16970d 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java @@ -321,9 +321,30 @@ public class HFileLink extends FileLink { public static boolean create(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final HRegionInfo hfileRegionInfo, final String hfileName) throws IOException { + return create(conf, fs, dstFamilyPath, hfileRegionInfo, hfileName, true); + } + + /** + * Create a new HFileLink + * + *
It also adds a back-reference to the hfile back-reference directory + * to simplify the reference-count and the cleaning process. + * + * @param conf {@link Configuration} to read for the archive directory name + * @param fs {@link FileSystem} on which to write the HFileLink + * @param dstFamilyPath - Destination path (table/region/cf/) + * @param hfileRegionInfo - Linked HFile Region Info + * @param hfileName - Linked HFile name + * @param createBackRef - Whether back reference should be created. Defaults to true. + * @return true if the file is created, otherwise the file exists. + * @throws IOException on file or parent directory creation failure + */ + public static boolean create(final Configuration conf, final FileSystem fs, + final Path dstFamilyPath, final HRegionInfo hfileRegionInfo, + final String hfileName, final boolean createBackRef) throws IOException { TableName linkedTable = hfileRegionInfo.getTable(); String linkedRegion = hfileRegionInfo.getEncodedName(); - return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName); + return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, createBackRef); } /** @@ -344,6 +365,28 @@ public class HFileLink extends FileLink { public static boolean create(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion, final String hfileName) throws IOException { + return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, true); + } + + /** + * Create a new HFileLink + * + *
It also adds a back-reference to the hfile back-reference directory + * to simplify the reference-count and the cleaning process. + * + * @param conf {@link Configuration} to read for the archive directory name + * @param fs {@link FileSystem} on which to write the HFileLink + * @param dstFamilyPath - Destination path (table/region/cf/) + * @param linkedTable - Linked Table Name + * @param linkedRegion - Linked Region Name + * @param hfileName - Linked HFile name + * @param createBackRef - Whether back reference should be created. Defaults to true. + * @return true if the file is created, otherwise the file exists. + * @throws IOException on file or parent directory creation failure + */ + public static boolean create(final Configuration conf, final FileSystem fs, + final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion, + final String hfileName, final boolean createBackRef) throws IOException { String familyName = dstFamilyPath.getName(); String regionName = dstFamilyPath.getParent().getName(); String tableName = FSUtils.getTableName(dstFamilyPath.getParent().getParent()) @@ -358,19 +401,24 @@ public class HFileLink extends FileLink { // Make sure the FileLink reference directory exists Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, linkedTable, linkedRegion, familyName); - Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName); - fs.mkdirs(backRefssDir); - - // Create the reference for the link - Path backRefPath = new Path(backRefssDir, refName); - fs.createNewFile(backRefPath); + Path backRefPath = null; + if (createBackRef) { + Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName); + fs.mkdirs(backRefssDir); + + // Create the reference for the link + backRefPath = new Path(backRefssDir, refName); + fs.createNewFile(backRefPath); + } try { // Create the link return fs.createNewFile(new Path(dstFamilyPath, name)); } catch (IOException e) { LOG.error("couldn't create the link=" + name + " for " + dstFamilyPath, e); // Revert the reference if the link creation failed - fs.delete(backRefPath, false); + if (createBackRef) { + fs.delete(backRefPath, false); + } throw e; } } @@ -389,13 +437,34 @@ public class HFileLink extends FileLink { * @throws IOException on file or parent directory creation failure */ public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs, - final Path dstFamilyPath, final String hfileLinkName) throws IOException { + final Path dstFamilyPath, final String hfileLinkName) + throws IOException { + return createFromHFileLink(conf, fs, dstFamilyPath, hfileLinkName, true); + } + + /** + * Create a new HFileLink starting from a hfileLink name + * + *
It also adds a back-reference to the hfile back-reference directory
+ * to simplify the reference-count and the cleaning process.
+ *
+ * @param conf {@link Configuration} to read for the archive directory name
+ * @param fs {@link FileSystem} on which to write the HFileLink
+ * @param dstFamilyPath - Destination path (table/region/cf/)
+ * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
+ * @param createBackRef - Whether back reference should be created. Defaults to true.
+ * @return true if the file is created, otherwise the file exists.
+ * @throws IOException on file or parent directory creation failure
+ */
+ public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs,
+ final Path dstFamilyPath, final String hfileLinkName, final boolean createBackRef)
+ throws IOException {
Matcher m = LINK_NAME_PATTERN.matcher(hfileLinkName);
if (!m.matches()) {
throw new IllegalArgumentException(hfileLinkName + " is not a valid HFileLink name!");
}
return create(conf, fs, dstFamilyPath, TableName.valueOf(m.group(1), m.group(2)),
- m.group(3), m.group(4));
+ m.group(3), m.group(4), createBackRef);
}
/**
diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
index 8e7a222..1c684fb 100644
--- hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -127,6 +127,7 @@ public class RestoreSnapshotHelper {
private final Configuration conf;
private final FileSystem fs;
+ private final boolean createBackRefs;
public RestoreSnapshotHelper(final Configuration conf,
final FileSystem fs,
@@ -134,7 +135,18 @@ public class RestoreSnapshotHelper {
final HTableDescriptor tableDescriptor,
final Path rootDir,
final ForeignExceptionDispatcher monitor,
- final MonitoredTask status)
+ final MonitoredTask status) {
+ this(conf, fs, manifest, tableDescriptor, rootDir, monitor, status, true);
+ }
+
+ public RestoreSnapshotHelper(final Configuration conf,
+ final FileSystem fs,
+ final SnapshotManifest manifest,
+ final HTableDescriptor tableDescriptor,
+ final Path rootDir,
+ final ForeignExceptionDispatcher monitor,
+ final MonitoredTask status,
+ final boolean createBackRefs)
{
this.fs = fs;
this.conf = conf;
@@ -146,6 +158,7 @@ public class RestoreSnapshotHelper {
this.tableDir = FSUtils.getTableDir(rootDir, tableDesc.getTableName());
this.monitor = monitor;
this.status = status;
+ this.createBackRefs = createBackRefs;
}
/**
@@ -484,7 +497,7 @@ public class RestoreSnapshotHelper {
for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) {
LOG.debug("Adding HFileLink " + storeFile.getName() +
" to region=" + regionInfo.getEncodedName() + " table=" + tableName);
- restoreStoreFile(familyDir, regionInfo, storeFile);
+ restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
}
} else {
// Family doesn't exists in the snapshot
@@ -505,7 +518,7 @@ public class RestoreSnapshotHelper {
for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) {
LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
- restoreStoreFile(familyDir, regionInfo, storeFile);
+ restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
}
}
}
@@ -598,7 +611,7 @@ public class RestoreSnapshotHelper {
Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
- restoreStoreFile(familyDir, snapshotRegionInfo, storeFile);
+ restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
}
}
}
@@ -630,17 +643,19 @@ public class RestoreSnapshotHelper {
*
* @param familyDir destination directory for the store file
* @param regionInfo destination region info for the table
+ * @param createBackRef - Whether back reference should be created. Defaults to true.
* @param storeFile store file name (can be a Reference, HFileLink or simple HFile)
*/
private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
- final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef)
+ throws IOException {
String hfileName = storeFile.getName();
if (HFileLink.isHFileLink(hfileName)) {
- HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName);
+ HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef);
} else if (StoreFileInfo.isReference(hfileName)) {
restoreReferenceFile(familyDir, regionInfo, storeFile);
} else {
- HFileLink.create(conf, fs, familyDir, regionInfo, hfileName);
+ HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef);
}
}
@@ -806,8 +821,10 @@ public class RestoreSnapshotHelper {
"Restoring snapshot '" + snapshotName + "' to directory " + restoreDir);
ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
+ // we send createBackRefs=false so that restored hfiles do not create back reference links
+ // in the base hbase root dir.
RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs,
- manifest, manifest.getTableDescriptor(), restoreDir, monitor, status);
+ manifest, manifest.getTableDescriptor(), restoreDir, monitor, status, false);
helper.restoreHdfsRegions(); // TODO: parallelize.
if (LOG.isDebugEnabled()) {
diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
index 347cad5..a936fc2 100644
--- hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
@@ -33,7 +32,6 @@ import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HConstants;
@@ -44,10 +42,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.master.AssignmentManager;
-import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
-import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
-import org.apache.hadoop.hbase.wal.WAL;
-import org.apache.hadoop.hbase.wal.WALFactory;
/**
* Utility methods for interacting with the regions.
@@ -176,11 +170,7 @@ public abstract class ModifyRegionUtils {
// unless I pass along via the conf.
Configuration confForWAL = new Configuration(conf);
confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
- WAL wal = (new WALFactory(confForWAL,
- Collections.