diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/migration/UpgradeTo96.java hbase-server/src/main/java/org/apache/hadoop/hbase/migration/UpgradeTo96.java index 40c2d8b..350c98d 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/migration/UpgradeTo96.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/migration/UpgradeTo96.java @@ -120,12 +120,12 @@ public class UpgradeTo96 extends Configured implements Tool { if (res == 0) LOG.info("No HFileV1 found."); else { LOG.warn("There are some HFileV1, or corrupt files (files with incorrect major version)."); - return -1; } + return res; } // if the user wants to upgrade, check for any HBase live process. // If yes, prompt the user to stop them - if (upgrade) { + else if (upgrade) { if (isAnyHBaseProcessAlive()) { LOG.error("Some HBase processes are still alive, or znodes not expired yet. " + "Please stop them before upgrade or try after some time."); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java index 0522de9..10c5b02 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.util; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -45,6 +46,9 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.io.FileLink; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; @@ -67,6 +71,8 @@ public class HFileV1Detector extends Configured implements Tool { private FileSystem fs; private static final Log LOG = LogFactory.getLog(HFileV1Detector.class); private static final int DEFAULT_NUM_OF_THREADS = 10; + private static final String DOT_ARCHIVE = ".archive"; + private static final String DOT_TMP = ".tmp"; private int numOfThreads; /** * directory to start the processing. @@ -93,6 +99,10 @@ public class HFileV1Detector extends Configured implements Tool { .newSetFromMap(new ConcurrentHashMap()); private Options options = new Options(); + /** + * used for computing pre-namespace paths for hfilelinks + */ + private Path defaultNamespace; public HFileV1Detector() { Option pathOption = new Option("p", "path", true, "Path to a table, or hbase installation"); @@ -171,6 +181,11 @@ public class HFileV1Detector extends Configured implements Tool { return -1; } + private void setDefaultNamespaceDir() throws IOException { + Path dataDir = new Path(FSUtils.getRootDir(getConf()), HConstants.BASE_NAMESPACE_DIR); + defaultNamespace = new Path(dataDir, NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR); + } + private int processResult(Set regionsWithHFileV1) { LOG.info("Result: \n"); printSet(processedTables, "Tables Processed: "); @@ -292,9 +307,9 @@ public class HFileV1Detector extends Configured implements Tool { if (StoreFileInfo.isReference(storeFilePath)) continue; // check whether this path is a HFileLink. else if (HFileLink.isHFileLink(storeFilePath)) { - HFileLink fileLink = new HFileLink(getConf(), storeFilePath); - fsdis = fileLink.open(fs); - lenToRead = fileLink.getFileStatus(fs).getLen(); + FileLink fLink = getFileLinkWithPreNSPath(storeFilePath); + fsdis = fLink.open(fs); + lenToRead = fLink.getFileStatus(fs).getLen(); } else { // a regular hfile fsdis = fs.open(storeFilePath); @@ -337,6 +352,45 @@ public class HFileV1Detector extends Configured implements Tool { return f; } + /** + * @param storeFilePath + * @return a FileLink which could read from pre-namespace paths. + * @throws IOException + */ + public FileLink getFileLinkWithPreNSPath(Path storeFilePath) throws IOException { + HFileLink link = new HFileLink(getConf(), storeFilePath); + List pathsToProcess = getPreNSPathsForHFileLink(link); + pathsToProcess.addAll(Arrays.asList(link.getLocations())); + return new FileLink(pathsToProcess); + } + + private List getPreNSPathsForHFileLink(HFileLink fileLink) throws IOException { + if (defaultNamespace == null) setDefaultNamespaceDir(); + List p = new ArrayList(); + String relativeTablePath = removeDefaultNSPath(fileLink.getOriginPath()); + p.add(getPreNSPath(DOT_ARCHIVE, relativeTablePath)); + p.add(getPreNSPath(DOT_TMP, relativeTablePath)); + p.add(getPreNSPath(null, relativeTablePath)); + return p; + } + + /** + * Removes the prefix of defaultNamespace from the path. + * @param originPath + * @return + */ + private String removeDefaultNSPath(Path originalPath) { + String pathStr = originalPath.toString(); + if (!pathStr.startsWith(defaultNamespace.toString())) return pathStr; + return pathStr.substring(defaultNamespace.toString().length() + 1); + } + + private Path getPreNSPath(String prefix, String relativeTablePath) throws IOException { + String relativePath = (prefix == null ? relativeTablePath : prefix + Path.SEPARATOR + + relativeTablePath); + return new Path(FSUtils.getRootDir(getConf()), relativePath); + } + private static boolean isTableDir(final FileSystem fs, final Path path) throws IOException { // check for old format, of having /table/.tableinfo; hbase:meta doesn't has .tableinfo, // include it. diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java index b18c2d4..e027706 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java @@ -17,24 +17,22 @@ */ package org.apache.hadoop.hbase.migration; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.MediumTests; -import org.apache.hadoop.hbase.master.TableNamespaceManager; +import org.apache.hadoop.hbase.io.FileLink; +import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.ReplicationPeer; @@ -43,20 +41,14 @@ import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HFileV1Detector; -import org.apache.hadoop.hbase.util.ZKDataMigrator; -import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.util.ToolRunner; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.server.ZKDatabase; -import org.junit.After; import org.junit.AfterClass; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.mockito.Mockito; import com.google.protobuf.InvalidProtocolBufferException; @@ -198,6 +190,35 @@ public class TestUpgradeTo96 { } @Test + public void testHFileLink() throws Exception { + // pass a link, and verify that correct paths are returned. + Path rootDir = FSUtils.getRootDir(TEST_UTIL.getConfiguration()); + Path aFileLink = new Path(rootDir, "table/2086db948c48/cf/table=21212abcdc33-0906db948c48"); + Path preNamespaceTablePath = new Path(rootDir, "table/21212abcdc33/cf/0906db948c48"); + Path preNamespaceArchivePath = + new Path(rootDir, ".archive/table/21212abcdc33/cf/0906db948c48"); + Path preNamespaceTempPath = new Path(rootDir, ".tmp/table/21212abcdc33/cf/0906db948c48"); + boolean preNSTablePathExists = false; + boolean preNSArchivePathExists = false; + boolean preNSTempPathExists = false; + assertTrue(HFileLink.isHFileLink(aFileLink)); + HFileLink hFileLink = new HFileLink(TEST_UTIL.getConfiguration(), aFileLink); + assertTrue(hFileLink.getArchivePath().toString().startsWith(rootDir.toString())); + + HFileV1Detector t = new HFileV1Detector(); + t.setConf(TEST_UTIL.getConfiguration()); + FileLink fileLink = t.getFileLinkWithPreNSPath(aFileLink); + //assert it has 6 paths (2 NS, 2 Pre NS, and 2 .tmp) to look. + assertTrue(fileLink.getLocations().length == 6); + for (Path p : fileLink.getLocations()) { + if (p.equals(preNamespaceArchivePath)) preNSArchivePathExists = true; + if (p.equals(preNamespaceTablePath)) preNSTablePathExists = true; + if (p.equals(preNamespaceTempPath)) preNSTempPathExists = true; + } + assertTrue(preNSArchivePathExists & preNSTablePathExists & preNSTempPathExists); + } + + @Test public void testADirForHFileV1() throws Exception { Path tablePath = new Path(hbaseRootDir, "foo"); System.out.println("testADirForHFileV1: " + tablePath.makeQualified(fs));