diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 0c439c7..2a0b0bb 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -274,17 +274,16 @@ public final class HConstants { /** Default HBase client operation timeout, which is tantamount to a blocking call */ public static final int DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT = Integer.MAX_VALUE; - /** Used to construct the name of the log directory for a region server - * Use '.' as a special character to seperate the log files from table data */ - public static final String HREGION_LOGDIR_NAME = ".logs"; + /** Used to construct the name of the log directory for a region server */ + public static final String HREGION_LOGDIR_NAME = "WALs"; /** Used to construct the name of the splitlog directory for a region server */ - public static final String SPLIT_LOGDIR_NAME = "splitlog"; - - public static final String CORRUPT_DIR_NAME = ".corrupt"; + public static final String SPLIT_LOGDIR_NAME = "splitWAL"; /** Like the previous, but for old logs that are about to be deleted */ - public static final String HREGION_OLDLOGDIR_NAME = ".oldlogs"; + public static final String HREGION_OLDLOGDIR_NAME = "oldWALs"; + + public static final String CORRUPT_DIR_NAME = "corrupt"; /** Used by HBCK to sideline backup data */ public static final String HBCK_SIDELINEDIR_NAME = ".hbck"; @@ -352,7 +351,7 @@ public final class HConstants { // be the first to be reassigned if the server(s) they are being served by // should go down. - public static final String BASE_NAMESPACE_DIR = ".data"; + public static final String BASE_NAMESPACE_DIR = "data"; /** delimiter used between portions of a region name */ public static final int META_ROW_DELIMITER = ','; @@ -815,7 +814,7 @@ public final class HConstants { public static final int REPLAY_QOS = 6; // REPLICATION_QOS < REPLAY_QOS < high_QOS /** Directory under /hbase where archived hfiles are stored */ - public static final String HFILE_ARCHIVE_DIRECTORY = ".archive"; + public static final String HFILE_ARCHIVE_DIRECTORY = "archive"; /** * Name of the directory to store all snapshots. See SnapshotDescriptionUtils for diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/migration/NamespaceUpgrade.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/migration/NamespaceUpgrade.java index 9b0107d..d50cf15 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/migration/NamespaceUpgrade.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/migration/NamespaceUpgrade.java @@ -19,7 +19,10 @@ */ package org.apache.hadoop.hbase.migration; -import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -36,8 +39,7 @@ import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.util.Tool; -import java.io.IOException; -import java.util.List; +import com.google.common.collect.Lists; /** * Upgrades old 0.94 filesystem layout to namespace layout @@ -46,6 +48,8 @@ import java.util.List; * - creates system namespace directory and move .META. table there * renaming .META. table to hbase:meta, * this in turn would require to re-encode the region directory name + * + *
The pre-0.96 paths and dir names are hardcoded in here.
*/
public class NamespaceUpgrade implements Tool {
private static final Log LOG = LogFactory.getLog(NamespaceUpgrade.class);
@@ -58,26 +62,33 @@ public class NamespaceUpgrade implements Tool {
private Path sysNsDir;
private Path defNsDir;
private Path baseDirs[];
+ // First move everything to this tmp .data dir in case there is a table named 'data'
+ private final String TMP_DATA_DIR = ".data";
+ // Old dir names to migrate.
+ private final String DOT_LOGS = ".logs";
+ private final String DOT_OLD_LOGS = ".oldlogs";
+ private final String DOT_CORRUPT = ".corrupt";
+ private final String DOT_SPLITLOG = "splitlog";
+ private final String DOT_ARCHIVE = ".archive";
public NamespaceUpgrade() throws IOException {
+ super();
}
public void init() throws IOException {
this.rootDir = FSUtils.getRootDir(conf);
this.fs = FileSystem.get(conf);
- sysNsDir = FSUtils.getNamespaceDir(rootDir, NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR);
- defNsDir = FSUtils.getNamespaceDir(rootDir, NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR);
+ Path tmpDataDir = new Path(rootDir, TMP_DATA_DIR);
+ sysNsDir = new Path(tmpDataDir, NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR);
+ defNsDir = new Path(tmpDataDir, NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR);
baseDirs = new Path[]{rootDir,
new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY),
new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY)};
}
- public void upgradeTableDirs()
- throws IOException, DeserializationException {
-
-
- //if new version is written then upgrade is done
+ public void upgradeTableDirs() throws IOException, DeserializationException {
+ // if new version is written then upgrade is done
if (verifyNSUpgrade(fs, rootDir)) {
return;
}
@@ -90,9 +101,75 @@ public class NamespaceUpgrade implements Tool {
migrateMeta();
+ migrateDotDirs();
+
+ deleteRoot();
+
FSUtils.setVersion(fs, rootDir);
}
+ /**
+ * Remove the -ROOT- dir. No longer of use.
+ * @throws IOException
+ */
+ public void deleteRoot() throws IOException {
+ Path rootDir = new Path(this.rootDir, "-ROOT-");
+ if (this.fs.exists(rootDir)) {
+ if (!this.fs.delete(rootDir, true)) LOG.info("Failed remove of " + rootDir);
+ LOG.info("Deleted " + rootDir);
+ }
+ }
+
+ /**
+ * Rename all the dot dirs -- .data, .archive, etc. -- as data, archive, etc.; i.e. minus the dot.
+ * @throws IOException
+ */
+ public void migrateDotDirs() throws IOException {
+ // Dot dirs to rename. Leave the tmp dir named '.tmp' and snapshots as .hbase-snapshot.
+ final Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
+ Path [][] dirs = new Path[][] {
+ new Path [] {new Path(rootDir, DOT_CORRUPT), new Path(rootDir, HConstants.CORRUPT_DIR_NAME)},
+ new Path [] {new Path(rootDir, DOT_LOGS), new Path(rootDir, HConstants.HREGION_LOGDIR_NAME)},
+ new Path [] {new Path(rootDir, DOT_OLD_LOGS),
+ new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME)},
+ new Path [] {new Path(rootDir, TMP_DATA_DIR),
+ new Path(rootDir, HConstants.BASE_NAMESPACE_DIR)}};
+ for (Path [] dir: dirs) {
+ Path src = dir[0];
+ Path tgt = dir[1];
+ if (!this.fs.exists(src)) {
+ LOG.info("Does not exist: " + src);
+ continue;
+ }
+ rename(src, tgt);
+ }
+ // Do the .archive dir. Need to move its subdirs to the default ns dir under data dir... so
+ // from '.archive/foo', to 'archive/data/default/foo'.
+ Path oldArchiveDir = new Path(rootDir, DOT_ARCHIVE);
+ if (this.fs.exists(oldArchiveDir)) {
+ // This is a pain doing two nn calls but portable over h1 and h2.
+ mkdirs(archiveDir);
+ Path archiveDataDir = new Path(archiveDir, HConstants.BASE_NAMESPACE_DIR);
+ mkdirs(archiveDataDir);
+ rename(oldArchiveDir, new Path(archiveDataDir,
+ NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR));
+ }
+ }
+
+ private void mkdirs(final Path p) throws IOException {
+ if (!this.fs.mkdirs(p)) throw new IOException("Failed make of " + p);
+ }
+
+ private void rename(final Path src, final Path tgt) throws IOException {
+ if (!fs.rename(src, tgt)) {
+ throw new IOException("Failed move " + src + " to " + tgt);
+ }
+ }
+
+ /**
+ * Create the system and default namespaces dirs
+ * @throws IOException
+ */
public void makeNamespaceDirs() throws IOException {
if (!fs.exists(sysNsDir)) {
if (!fs.mkdirs(sysNsDir)) {
@@ -106,27 +183,45 @@ public class NamespaceUpgrade implements Tool {
}
}
+ /**
+ * Migrate all tables into respective namespaces, either default or system. We put them into
+ * a temporary location, '.data', in case a user table is name 'data'. In a later method we will
+ * move stuff from .data to data.
+ * @throws IOException
+ */
public void migrateTables() throws IOException {
List Uses a tarball which is an image of an 0.94 hbase.rootdir.
*
- * Contains tables with currentKeys as the stored keys:
+ * Contains tables with currentKeys as the stored keys:
* foo, ns1.foo, ns2.foo
*
- * Contains snapshots with snapshot{num}Keys as the contents:
+ * Contains snapshots with snapshot{num}Keys as the contents:
* snapshot1Keys, snapshot2Keys
*
*/
@@ -76,6 +77,7 @@ public class TestNamespaceUpgrade {
private final static String currentKeys[] =
{"1","2","3","4","5","6","7","8","9","A"};
private final static String tables[] = {"foo", "ns1.foo","ns.two.foo"};
+ private FsShell shell;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
@@ -103,7 +105,6 @@ public class TestNamespaceUpgrade {
Configuration toolConf = TEST_UTIL.getConfiguration();
conf.set(HConstants.HBASE_DIR, TEST_UTIL.getDefaultRootDirPath().toString());
ToolRunner.run(toolConf, new NamespaceUpgrade(), new String[]{"--upgrade"});
-
assertTrue(FSUtils.getVersion(fs, hbaseRootDir).equals(HConstants.FILE_SYSTEM_VERSION));
TEST_UTIL.startMiniHBaseCluster(1, 1);
@@ -154,7 +155,7 @@ public class TestNamespaceUpgrade {
@Test
public void testSnapshots() throws IOException, InterruptedException {
String snapshots[][] = {snapshot1Keys, snapshot2Keys};
- for(int i=1; i<=snapshots.length; i++) {
+ for(int i = 1; i <= snapshots.length; i++) {
for(String table: tables) {
TEST_UTIL.getHBaseAdmin().cloneSnapshot(table+"_snapshot"+i, table+"_clone"+i);
FSUtils.logFileSystemState(FileSystem.get(TEST_UTIL.getConfiguration()),
@@ -171,7 +172,7 @@ public class TestNamespaceUpgrade {
}
@Test
- public void testRenameUsingSnapshots() throws IOException, InterruptedException {
+ public void testRenameUsingSnapshots() throws Exception {
String newNS = "newNS";
TEST_UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(newNS).build());
for(String table: tables) {
@@ -180,10 +181,9 @@ public class TestNamespaceUpgrade {
Scan())) {
assertEquals(currentKeys[count++], Bytes.toString(res.getRow()));
}
- TEST_UTIL.getHBaseAdmin().snapshot(table+"_snapshot3", table);
- final String newTableName =
- newNS+ TableName.NAMESPACE_DELIM+table+"_clone3";
- TEST_UTIL.getHBaseAdmin().cloneSnapshot(table+"_snapshot3", newTableName);
+ TEST_UTIL.getHBaseAdmin().snapshot(table + "_snapshot3", table);
+ final String newTableName = newNS + TableName.NAMESPACE_DELIM + table + "_clone3";
+ TEST_UTIL.getHBaseAdmin().cloneSnapshot(table + "_snapshot3", newTableName);
Thread.sleep(1000);
count = 0;
for(Result res: new HTable(TEST_UTIL.getConfiguration(), newTableName).getScanner(new
@@ -224,7 +224,5 @@ public class TestNamespaceUpgrade {
}
Assert.assertEquals(newTableName, currentKeys.length, count);
}
-
}
-}
-
+}
\ No newline at end of file
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index 5b148a7..4226e94 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -726,17 +726,18 @@ public class TestHLog {
Assert.assertNull(HLogUtil.getServerNameFromHLogDirectoryName(conf, hl + "qdf"));
Assert.assertNull(HLogUtil.getServerNameFromHLogDirectoryName(conf, "sfqf" + hl + "qdf"));
+ final String wals = "/WALs/";
ServerName parsed = HLogUtil.getServerNameFromHLogDirectoryName(conf,
- FSUtils.getRootDir(conf).toUri().toString() +
- "/.logs/" + sn + "/localhost%2C32984%2C1343316388997.1343316390417");
+ FSUtils.getRootDir(conf).toUri().toString() + wals + sn +
+ "/localhost%2C32984%2C1343316388997.1343316390417");
Assert.assertEquals("standard", sn, parsed);
parsed = HLogUtil.getServerNameFromHLogDirectoryName(conf, hl + "/qdf");
Assert.assertEquals("subdir", sn, parsed);
parsed = HLogUtil.getServerNameFromHLogDirectoryName(conf,
- FSUtils.getRootDir(conf).toUri().toString() +
- "/.logs/" + sn + "-splitting/localhost%3A57020.1340474893931");
+ FSUtils.getRootDir(conf).toUri().toString() + wals + sn +
+ "-splitting/localhost%3A57020.1340474893931");
Assert.assertEquals("split", sn, parsed);
}