diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 3ec7435..4aaec86 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -274,17 +274,16 @@ public final class HConstants { /** Default HBase client operation timeout, which is tantamount to a blocking call */ public static final int DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT = Integer.MAX_VALUE; - /** Used to construct the name of the log directory for a region server - * Use '.' as a special character to seperate the log files from table data */ - public static final String HREGION_LOGDIR_NAME = ".logs"; + /** Used to construct the name of the log directory for a region server */ + public static final String HREGION_LOGDIR_NAME = "WALs"; /** Used to construct the name of the splitlog directory for a region server */ - public static final String SPLIT_LOGDIR_NAME = "splitlog"; - - public static final String CORRUPT_DIR_NAME = ".corrupt"; + public static final String SPLIT_LOGDIR_NAME = "splitWAL"; /** Like the previous, but for old logs that are about to be deleted */ - public static final String HREGION_OLDLOGDIR_NAME = ".oldlogs"; + public static final String HREGION_OLDLOGDIR_NAME = "oldWALs"; + + public static final String CORRUPT_DIR_NAME = "corrupt"; /** Used by HBCK to sideline backup data */ public static final String HBCK_SIDELINEDIR_NAME = ".hbck"; @@ -352,7 +351,7 @@ public final class HConstants { // be the first to be reassigned if the server(s) they are being served by // should go down. - public static final String BASE_NAMESPACE_DIR = ".data"; + public static final String BASE_NAMESPACE_DIR = "data"; /** delimiter used between portions of a region name */ public static final int META_ROW_DELIMITER = ','; @@ -805,7 +804,7 @@ public final class HConstants { public static final int REPLAY_QOS = 6; // REPLICATION_QOS < REPLAY_QOS < high_QOS /** Directory under /hbase where archived hfiles are stored */ - public static final String HFILE_ARCHIVE_DIRECTORY = ".archive"; + public static final String HFILE_ARCHIVE_DIRECTORY = "archive"; /** * Name of the directory to store all snapshots. See SnapshotDescriptionUtils for diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/migration/NamespaceUpgrade.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/migration/NamespaceUpgrade.java index 9b0107d..c118aea 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/migration/NamespaceUpgrade.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/migration/NamespaceUpgrade.java @@ -19,7 +19,10 @@ */ package org.apache.hadoop.hbase.migration; -import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -36,8 +39,7 @@ import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.util.Tool; -import java.io.IOException; -import java.util.List; +import com.google.common.collect.Lists; /** * Upgrades old 0.94 filesystem layout to namespace layout @@ -46,6 +48,8 @@ import java.util.List; * - creates system namespace directory and move .META. table there * renaming .META. table to hbase:meta, * this in turn would require to re-encode the region directory name + * + *

The pre-0.96 paths and dir names are hardcoded in here. */ public class NamespaceUpgrade implements Tool { private static final Log LOG = LogFactory.getLog(NamespaceUpgrade.class); @@ -58,26 +62,27 @@ public class NamespaceUpgrade implements Tool { private Path sysNsDir; private Path defNsDir; private Path baseDirs[]; + // First move everything to this tmp .data dir in case there is a table named 'data' + private final String TMP_DATA_DIR = ".data"; public NamespaceUpgrade() throws IOException { + super(); } public void init() throws IOException { this.rootDir = FSUtils.getRootDir(conf); this.fs = FileSystem.get(conf); - sysNsDir = FSUtils.getNamespaceDir(rootDir, NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR); - defNsDir = FSUtils.getNamespaceDir(rootDir, NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR); + Path tmpDataDir = new Path(rootDir, TMP_DATA_DIR); + sysNsDir = new Path(tmpDataDir, NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR); + defNsDir = new Path(tmpDataDir, NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR); baseDirs = new Path[]{rootDir, new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY), new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY)}; } - public void upgradeTableDirs() - throws IOException, DeserializationException { - - - //if new version is written then upgrade is done + public void upgradeTableDirs() throws IOException, DeserializationException { + // if new version is written then upgrade is done if (verifyNSUpgrade(fs, rootDir)) { return; } @@ -90,9 +95,67 @@ public class NamespaceUpgrade implements Tool { migrateMeta(); + migrateDotDirs(); + + deleteRoot(); + FSUtils.setVersion(fs, rootDir); } + /** + * Remove the -ROOT- dir. No longer of use. + * @throws IOException + */ + public void deleteRoot() throws IOException { + Path rootDir = new Path(this.rootDir, "-ROOT-"); + if (this.fs.exists(rootDir)) { + if (!this.fs.delete(rootDir, true)) LOG.info("Failed remove of " + rootDir); + LOG.info("Deleted " + rootDir); + } + } + + /** + * Rename all the dot dirs -- .data, .archive, etc. -- as data, archive, etc.; i.e. minus the dot. + * @throws IOException + */ + public void migrateDotDirs() throws IOException { + // Dot dirs to rename. Leave the tmp dir named '.tmp' and snapshots as .hbase-snapshot. + final Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); + Path [][] dirs = new Path[][] { + new Path [] {new Path(rootDir, ".corrupt"), new Path(rootDir, HConstants.CORRUPT_DIR_NAME)}, + new Path [] {new Path(rootDir, ".logs"), new Path(rootDir, HConstants.HREGION_LOGDIR_NAME)}, + new Path [] {new Path(rootDir, ".oldlogs"), + new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME)}, + new Path [] {new Path(rootDir, ".archive"), archiveDir}, + new Path [] {new Path(rootDir, TMP_DATA_DIR), + new Path(rootDir, HConstants.BASE_NAMESPACE_DIR)}}; + for (Path [] dir: dirs) { + Path src = dir[0]; + Path tgt = dir[1]; + if (!this.fs.exists(src)) { + LOG.info("Does not exist: " + src); + continue; + } + rename(src, tgt); + } + if (this.fs.exists(archiveDir)) { + // Rename the subdir name .data. + Path dotDataDir = new Path(archiveDir, ".data"); + Path dataDir = new Path(archiveDir, "data"); + if (this.fs.exists(dotDataDir)) rename(dotDataDir, dataDir); + } + } + + private void rename(final Path src, final Path tgt) throws IOException { + if (!fs.rename(src, tgt)) { + throw new IOException("Failed move " + src + " to "+tgt); + } + } + + /** + * Create the system and default namespaces dirs + * @throws IOException + */ public void makeNamespaceDirs() throws IOException { if (!fs.exists(sysNsDir)) { if (!fs.mkdirs(sysNsDir)) { @@ -106,27 +169,45 @@ public class NamespaceUpgrade implements Tool { } } + /** + * Migrate all tables into respective namespaces, either default or system. We put them into + * a temporary location, '.data', in case a user table is name 'data'. In a later method we will + * move stuff from .data to data. + * @throws IOException + */ public void migrateTables() throws IOException { List sysTables = Lists.newArrayList("-ROOT-",".META."); + /** Directories that are not HBase table directories */ + final List nonUserTableDirs = Arrays.asList(new String[] { + ".logs", + ".oldlogs", + ".corrupt", + "splitlog", + HConstants.HBCK_SIDELINEDIR_NAME, + ".archive", + HConstants.SNAPSHOT_DIR_NAME, + HConstants.HBASE_TEMP_DIRECTORY, + ".data"}); - //migrate tables including archive and tmp - for(Path baseDir: baseDirs) { + // Migrate tables including archive and tmp + for (Path baseDir: baseDirs) { if (!fs.exists(baseDir)) continue; List oldTableDirs = FSUtils.getLocalTableDirs(fs, baseDir); - for(Path oldTableDir: oldTableDirs) { - if (!sysTables.contains(oldTableDir.getName())) { - Path nsDir = FSUtils.getTableDir(baseDir, - TableName.valueOf(oldTableDir.getName())); - if(!fs.exists(nsDir.getParent())) { - if(!fs.mkdirs(nsDir.getParent())) { - throw new IOException("Failed to create namespace dir "+nsDir.getParent()); - } + for (Path oldTableDir: oldTableDirs) { + if (nonUserTableDirs.contains(oldTableDir.getName())) continue; + if (sysTables.contains(oldTableDir.getName())) continue; + // Make the new directory under the ns to which we will move the table. + Path nsDir = new Path(this.defNsDir, + TableName.valueOf(oldTableDir.getName()).getQualifierAsString()); + if (!fs.exists(nsDir.getParent())) { + if (!fs.mkdirs(nsDir.getParent())) { + throw new IOException("Failed to create namespace dir "+nsDir.getParent()); } - if (sysTables.indexOf(oldTableDir.getName()) < 0) { - LOG.info("Migrating table " + oldTableDir.getName() + " to " + nsDir); - if (!fs.rename(oldTableDir, nsDir)) { - throw new IOException("Failed to move "+oldTableDir+" to namespace dir "+nsDir); - } + } + if (sysTables.indexOf(oldTableDir.getName()) < 0) { + LOG.info("Migrating table " + oldTableDir.getName() + " to " + nsDir); + if (!fs.rename(oldTableDir, nsDir)) { + throw new IOException("Failed to move "+oldTableDir+" to namespace dir "+nsDir); } } } @@ -163,8 +244,9 @@ public class NamespaceUpgrade implements Tool { } public void migrateMeta() throws IOException { - Path newMetaRegionDir = HRegion.getRegionDir(rootDir, HRegionInfo.FIRST_META_REGIONINFO); - Path newMetaDir = FSUtils.getTableDir(rootDir, TableName.META_TABLE_NAME); + Path newMetaDir = new Path(this.sysNsDir, TableName.META_TABLE_NAME.getQualifierAsString()); + Path newMetaRegionDir = + new Path(newMetaDir, HRegionInfo.FIRST_META_REGIONINFO.getEncodedName()); Path oldMetaDir = new Path(rootDir, ".META."); if (fs.exists(oldMetaDir)) { LOG.info("Migrating meta table " + oldMetaDir.getName() + " to " + newMetaDir); @@ -174,10 +256,9 @@ public class NamespaceUpgrade implements Tool { } } - //since meta table name has changed - //rename meta region dir from it's old encoding to new one + // Since meta table name has changed rename meta region dir from it's old encoding to new one Path oldMetaRegionDir = HRegion.getRegionDir(rootDir, - new Path(newMetaDir, "1028785192").toString()); + new Path(newMetaDir, "1028785192").toString()); if (fs.exists(oldMetaRegionDir)) { LOG.info("Migrating meta region " + oldMetaRegionDir + " to " + newMetaRegionDir); if (!fs.rename(oldMetaRegionDir, newMetaRegionDir)) { @@ -199,7 +280,7 @@ public class NamespaceUpgrade implements Tool { @Override public int run(String[] args) throws Exception { - if(args.length < 1 || !args[0].equals("--upgrade")) { + if (args.length < 1 || !args[0].equals("--upgrade")) { System.out.println("Usage: --upgrade"); return 0; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestNamespaceUpgrade.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestNamespaceUpgrade.java index e3a3f46..eeff846 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestNamespaceUpgrade.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestNamespaceUpgrade.java @@ -24,8 +24,6 @@ import static org.junit.Assert.*; import java.io.File; import java.io.IOException; -import junit.framework.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -47,6 +45,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.util.ToolRunner; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -56,12 +55,12 @@ import org.junit.experimental.categories.Category; * Mainly tests that tables are migrated and consistent. Also verifies * that snapshots have been migrated correctly. * - * Uses a tarball which is an image of an 0.94 hbase.rootdir. + *

Uses a tarball which is an image of an 0.94 hbase.rootdir. * - * Contains tables with currentKeys as the stored keys: + *

Contains tables with currentKeys as the stored keys: * foo, ns1.foo, ns2.foo * - * Contains snapshots with snapshot{num}Keys as the contents: + *

Contains snapshots with snapshot{num}Keys as the contents: * snapshot1Keys, snapshot2Keys * */ @@ -103,7 +102,7 @@ public class TestNamespaceUpgrade { Configuration toolConf = TEST_UTIL.getConfiguration(); conf.set(HConstants.HBASE_DIR, TEST_UTIL.getDefaultRootDirPath().toString()); ToolRunner.run(toolConf, new NamespaceUpgrade(), new String[]{"--upgrade"}); - + doFsCommand(shell, new String [] {"-lsr", "/"}); assertTrue(FSUtils.getVersion(fs, hbaseRootDir).equals(HConstants.FILE_SYSTEM_VERSION)); TEST_UTIL.startMiniHBaseCluster(1, 1); @@ -154,7 +153,7 @@ public class TestNamespaceUpgrade { @Test public void testSnapshots() throws IOException, InterruptedException { String snapshots[][] = {snapshot1Keys, snapshot2Keys}; - for(int i=1; i<=snapshots.length; i++) { + for(int i = 1; i <= snapshots.length; i++) { for(String table: tables) { TEST_UTIL.getHBaseAdmin().cloneSnapshot(table+"_snapshot"+i, table+"_clone"+i); FSUtils.logFileSystemState(FileSystem.get(TEST_UTIL.getConfiguration()),