From 7074fb6a038cf5ba8e241ce5496b20d8d908ff49 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Tue, 29 Jul 2014 10:18:20 -0500 Subject: [PATCH] HBASE-4744 enable test that log roll fails after split start. Note that this changes testLogRollAfterSplitStart from a Large test to a Medium test. FSHLog changes: * make logging in replaceWriter handle null newPath * clean up javadoc for replaceWriter TestHLogSplit changes: * removed testLogRollAfterSplitStart TestLogRollAbort * added testLogRollAfterSplitStart * remove @Ignore, add debug messages about test progress * clean up check for proper failure from log roll --- .../hadoop/hbase/regionserver/wal/FSHLog.java | 28 ++++--- .../hbase/regionserver/wal/TestHLogSplit.java | 64 --------------- .../hbase/regionserver/wal/TestLogRollAbort.java | 86 ++++++++++++++++++++ 3 files changed, 104 insertions(+), 74 deletions(-) diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java index b816192..8474836 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java @@ -823,14 +823,21 @@ class FSHLog implements HLog, Syncable { /** * Cleans up current writer closing it and then puts in place the passed in - * nextWriter - * - * @param oldPath - * @param newPath - * @param nextWriter - * @param nextHdfsOut - * @return newPath - * @throws IOException + * nextWriter. + * + * In the case of creating a new WAL, oldPath will be null. + * + * In the case of rolling over from one file to the next, none of the params will be null. + * + * In the case of closing out this FSHLog with no further use newPath, nextWriter, and + * nextHdfsOut will be null. + * + * @param oldPath may be null + * @param newPath may be null + * @param nextWriter may be null + * @param nextHdfsOut may be null + * @return the passed in newPath + * @throws IOException if there is a problem flushing or closing the underlying FS */ Path replaceWriter(final Path oldPath, final Path newPath, FSHLog.Writer nextWriter, final FSDataOutputStream nextHdfsOut) @@ -885,6 +892,7 @@ class FSHLog implements HLog, Syncable { this.hdfs_out = nextHdfsOut; int oldNumEntries = this.numEntries.get(); this.numEntries.set(0); + final String newPathString = (null == newPath ? null : FSUtils.getPath(newPath)); if (oldPath != null) { this.byWalRegionSequenceIds.put(oldPath, this.highestRegionSequenceIds); this.highestRegionSequenceIds = new HashMap(); @@ -892,9 +900,9 @@ class FSHLog implements HLog, Syncable { this.totalLogSize.addAndGet(oldFileLen); LOG.info("Rolled WAL " + FSUtils.getPath(oldPath) + " with entries=" + oldNumEntries + ", filesize=" + StringUtils.byteDesc(oldFileLen) + "; new WAL " + - FSUtils.getPath(newPath)); + newPathString); } else { - LOG.info("New WAL " + FSUtils.getPath(newPath)); + LOG.info("New WAL " + newPathString); } } catch (InterruptedException ie) { // Perpetuate the interrupt diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java index dc39415..e7997de 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java @@ -1072,70 +1072,6 @@ public class TestHLogSplit { assertEquals(regions.size(), outputCounts.size()); } - // HBASE-2312: tests the case where a RegionServer enters a GC pause, - // comes back online after the master declared it dead and started to split. - // Want log rolling after a master split to fail - @Test (timeout=300000) - @Ignore("Need HADOOP-6886, HADOOP-6840, & HDFS-617 for this. HDFS 0.20.205.1+ should have this") - public void testLogRollAfterSplitStart() throws IOException { - HLog log = null; - String logName = "testLogRollAfterSplitStart"; - Path thisTestsDir = new Path(HBASEDIR, logName); - - try { - // put some entries in an HLog - TableName tableName = - TableName.valueOf(this.getClass().getName()); - HRegionInfo regioninfo = new HRegionInfo(tableName, - HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); - log = HLogFactory.createHLog(fs, HBASEDIR, logName, conf); - final AtomicLong sequenceId = new AtomicLong(1); - - final int total = 20; - for (int i = 0; i < total; i++) { - WALEdit kvs = new WALEdit(); - kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName())); - HTableDescriptor htd = new HTableDescriptor(tableName); - htd.addFamily(new HColumnDescriptor("column")); - log.append(regioninfo, tableName, kvs, System.currentTimeMillis(), htd, sequenceId); - } - // Send the data to HDFS datanodes and close the HDFS writer - log.sync(); - ((FSHLog) log).replaceWriter(((FSHLog)log).getOldPath(), null, null, null); - - /* code taken from ProcessServerShutdown.process() - * handles RS shutdowns (as observed by the Master) - */ - // rename the directory so a rogue RS doesn't create more HLogs - Path rsSplitDir = new Path(thisTestsDir.getParent(), - thisTestsDir.getName() + "-splitting"); - fs.rename(thisTestsDir, rsSplitDir); - LOG.debug("Renamed region directory: " + rsSplitDir); - - // Process the old log files - HLogSplitter.split(HBASEDIR, rsSplitDir, OLDLOGDIR, fs, conf); - - // Now, try to roll the HLog and verify failure - try { - log.rollWriter(); - Assert.fail("rollWriter() did not throw any exception."); - } catch (IOException ioe) { - if (ioe.getCause().getMessage().contains("FileNotFound")) { - LOG.info("Got the expected exception: ", ioe.getCause()); - } else { - Assert.fail("Unexpected exception: " + ioe); - } - } - } finally { - if (log != null) { - log.close(); - } - if (fs.exists(thisTestsDir)) { - fs.delete(thisTestsDir, true); - } - } - } - /** * This thread will keep adding new log files * It simulates a region server that was considered dead but woke up and wrote diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java index 79fdee2..2e0f2cc 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java @@ -17,14 +17,24 @@ */ package org.apache.hadoop.hbase.regionserver.wal; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.Assert; import static org.junit.Assert.assertTrue; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.MediumTests; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.TableName; @@ -53,6 +63,10 @@ public class TestLogRollAbort { private static MiniHBaseCluster cluster; private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + /* For the split-then-roll test */ + private static final Path HBASEDIR = new Path("/hbase"); + private static final Path OLDLOGDIR = new Path(HBASEDIR, "hlog.old"); + // Need to override this setup so we can edit the config before it gets sent // to the HDFS & HBase cluster startup. @BeforeClass @@ -76,6 +90,9 @@ public class TestLogRollAbort { TEST_UTIL.getConfiguration().setInt("dfs.client.block.write.retries", 10); } + private Configuration conf; + private FileSystem fs; + @Before public void setUp() throws Exception { TEST_UTIL.startMiniCluster(2); @@ -83,6 +100,8 @@ public class TestLogRollAbort { cluster = TEST_UTIL.getHBaseCluster(); dfsCluster = TEST_UTIL.getDFSCluster(); admin = TEST_UTIL.getHBaseAdmin(); + conf = TEST_UTIL.getConfiguration(); + fs = TEST_UTIL.getDFSCluster().getFileSystem(); // disable region rebalancing (interferes with log watching) cluster.getMaster().balanceSwitch(false); @@ -148,4 +167,71 @@ public class TestLogRollAbort { table.close(); } } + + /** + * Tests the case where a RegionServer enters a GC pause, + * comes back online after the master declared it dead and started to split. + * Want log rolling after a master split to fail. See HBASE-2312. + */ + @Test (timeout=300000) + public void testLogRollAfterSplitStart() throws IOException { + LOG.info("Verify wal roll after split starts will fail."); + HLog log = null; + String logName = "testLogRollAfterSplitStart"; + Path thisTestsDir = new Path(HBASEDIR, logName); + + try { + // put some entries in an HLog + TableName tableName = + TableName.valueOf(this.getClass().getName()); + HRegionInfo regioninfo = new HRegionInfo(tableName, + HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); + log = HLogFactory.createHLog(fs, HBASEDIR, logName, conf); + final AtomicLong sequenceId = new AtomicLong(1); + + final int total = 20; + for (int i = 0; i < total; i++) { + WALEdit kvs = new WALEdit(); + kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName())); + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor("column")); + log.append(regioninfo, tableName, kvs, System.currentTimeMillis(), htd, sequenceId); + } + // Send the data to HDFS datanodes and close the HDFS writer + log.sync(); + ((FSHLog) log).replaceWriter(((FSHLog)log).getOldPath(), null, null, null); + + /* code taken from MasterFileSystem.getLogDirs(), which is called from MasterFileSystem.splitLog() + * handles RS shutdowns (as observed by the splitting process) + */ + // rename the directory so a rogue RS doesn't create more HLogs + Path rsSplitDir = thisTestsDir.suffix(HLog.SPLITTING_EXT); + if (!fs.rename(thisTestsDir, rsSplitDir)) { + throw new IOException("Failed fs.rename for log split: " + thisTestsDir); + } + LOG.debug("Renamed region directory: " + rsSplitDir); + + LOG.debug("Processing the old log files."); + HLogSplitter.split(HBASEDIR, rsSplitDir, OLDLOGDIR, fs, conf); + + LOG.debug("Trying to roll the HLog."); + try { + log.rollWriter(); + Assert.fail("rollWriter() did not throw any exception."); + } catch (IOException ioe) { + if (ioe.getCause() instanceof FileNotFoundException) { + LOG.info("Got the expected exception: ", ioe.getCause()); + } else { + Assert.fail("Unexpected exception: " + ioe); + } + } + } finally { + if (log != null) { + log.close(); + } + if (fs.exists(thisTestsDir)) { + fs.delete(thisTestsDir, true); + } + } + } } -- 1.7.10.2 (Apple Git-33)