From c50858405e556c7957648daef17c98aad2403a16 Mon Sep 17 00:00:00 2001 From: Sakthi Date: Fri, 14 Dec 2018 18:30:53 -0800 Subject: [PATCH] HBASE-20984: Add/Modify test case to check custom hbase.wal.dir outside hdfs filesystem --- .../hadoop/hbase/HBaseTestingUtility.java | 17 ++++++++++++----- .../apache/hadoop/hbase/wal/TestWALFactory.java | 1 - .../apache/hadoop/hbase/wal/TestWALRootDir.java | 14 ++++++++++++++ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 31a7cad3f5ec5bff779cf752dadaac4dec7c6f8c..2cd93b8c23a72221446d1f885a6b48988f901587 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -551,16 +551,16 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { /** * Sets up a new path in test filesystem to be used by tests. */ - private Path getNewDataTestDirOnTestFS() throws IOException { + private Path getNewDataTestDirOnTestFS(FileSystem fs) throws IOException { //The file system can be either local, mini dfs, or if the configuration //is supplied externally, it can be an external cluster FS. If it is a local //file system, the tests should use getBaseTestDir, otherwise, we can use //the working directory, and create a unique sub dir there - FileSystem fs = getTestFileSystem(); + Path newDataTestDir; String randomStr = getRandomUUID().toString(); if (fs.getUri().getScheme().equals(FileSystem.getLocal(conf).getUri().getScheme())) { - newDataTestDir = new Path(getDataTestDir(), randomStr); + newDataTestDir = new Path("file:///"+getDataTestDir(), randomStr); File dataTestDir = new File(newDataTestDir.toString()); if (deleteOnExit()) dataTestDir.deleteOnExit(); } else { @@ -571,6 +571,10 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { return newDataTestDir; } + private Path getNewDataTestDirOnTestFS() throws IOException { + return getNewDataTestDirOnTestFS(getTestFileSystem()); + } + /** * Cleans the test data directory on the test filesystem. * @return True if we removed the test dirs @@ -1353,8 +1357,11 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { * @throws IOException */ public Path createWALRootDir() throws IOException { - FileSystem fs = FileSystem.get(this.conf); - Path walDir = getNewDataTestDirOnTestFS(); + return createWALRootDir(FileSystem.get(this.conf)); + } + + public Path createWALRootDir(FileSystem fs) throws IOException { + Path walDir = getNewDataTestDirOnTestFS(fs); FSUtils.setWALRootDir(this.conf, walDir); fs.mkdirs(walDir); return walDir; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java index 8fbe09dd30bac8f12165b1aa9964054041bfd2f4..587198160ffc1b6371bc56dab79dbc415ef67067 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java @@ -130,7 +130,6 @@ public class TestWALFactory { @BeforeClass public static void setUpBeforeClass() throws Exception { - CommonFSUtils.setWALRootDir(TEST_UTIL.getConfiguration(), new Path("file:///tmp/wal")); // Make block sizes small. TEST_UTIL.getConfiguration().setInt("dfs.blocksize", 1024 * 1024); // needed for testAppendClose() diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALRootDir.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALRootDir.java index 40fad6ad5208c8ac0b728923b7e42d97ec8779db..0a1cd2ca57383aa02b4ab40d91edc8b1cf960166 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALRootDir.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALRootDir.java @@ -26,6 +26,7 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -88,7 +89,20 @@ public class TestWALRootDir { } @Test + public void testWALRootDirInMultipleFS() throws Exception { + // Test WALRootDir in dfs + testWALRootDir(); + cleanup(); + + // Test WALRootDir in local fs + walRootDir = TEST_UTIL.createWALRootDir(new LocalFileSystem()); + walFs = FSUtils.getWALFileSystem(conf); + testWALRootDir(); + } + public void testWALRootDir() throws Exception { + LOG.debug("Current HBase Root Dir is {}", rootDir); + LOG.debug("Current WAL Root Dir is {}", walRootDir); RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build(); wals = new WALFactory(conf, "testWALRootDir"); WAL log = wals.getWAL(regionInfo); -- 2.17.2 (Apple Git-113)