Index: src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java (revision 1097342) +++ src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java (working copy) @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.regionserver.wal.OrphanHLogAfterSplitException; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hdfs.server.namenode.SafeModeException; /** * This class abstracts a bunch of operations the HMaster needs to interact with @@ -205,8 +206,12 @@ splitTime = splitter.getTime(); splitLogSize = splitter.getSize(); } catch (IOException e) { - checkFileSystem(); - LOG.error("Failed splitting " + logDir.toString(), e); + if (e instanceof SafeModeException) { + this.master.abort("Shutting down HBase cluster: Failed splitting hlog files.", e); + } else { + checkFileSystem(); + LOG.error("Failed splitting " + logDir.toString(), e); + } } finally { this.splitLogLock.unlock(); } Index: src/main/java/org/apache/hadoop/hbase/util/FSUtils.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/util/FSUtils.java (revision 1097342) +++ src/main/java/org/apache/hadoop/hbase/util/FSUtils.java (working copy) @@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException; +import org.apache.hadoop.hdfs.server.namenode.SafeModeException; import org.apache.hadoop.io.SequenceFile; import java.io.DataInputStream; @@ -660,7 +661,8 @@ } LOG.info("Recovering file " + p); long startWaiting = System.currentTimeMillis(); - + long maxWaitTime = startWaiting + conf.getLong("hbase.master.safemodeWaitTimeout", 120000); + // Trying recovery boolean recovered = false; while (!recovered) { @@ -700,6 +702,13 @@ // This exception comes out instead of FNFE, fix it throw new FileNotFoundException( "The given HLog wasn't found at " + p.toString()); + } else if (e instanceof SafeModeException) { + long now = System.currentTimeMillis(); + if (now >= maxWaitTime) { + throw e; + } + LOG.warn("Namenode is not exit SafeMode; waiting up to " + + (maxWaitTime - now) + "ms", e); } else { throw new IOException("Failed to open " + p + " for append", e); }