Index: src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java (revision 1240082) +++ src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java (working copy) @@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.regionserver.wal.HLogLengthMisMatchException; import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter; import org.apache.hadoop.hbase.regionserver.wal.OrphanHLogAfterSplitException; import org.apache.hadoop.hbase.util.Bytes; @@ -214,7 +215,8 @@ splitLogSize = splitter.getSize(); retrySplitting = false; } catch (IOException e) { - if (checkFileSystem() && retrySplitting) { + if (checkFileSystem() && retrySplitting + && !(e instanceof HLogLengthMisMatchException)) { LOG.info("Retrying failed log splitting " + logDir.toString()); } else { LOG.fatal("Failed splitting " + logDir.toString(), e); @@ -222,6 +224,7 @@ "Shutting down HBase cluster: file system not available", e); } } + } while (retrySplitting); } finally { this.splitLogLock.unlock(); Index: src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (revision 1240082) +++ src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (working copy) @@ -633,6 +633,10 @@ throw e; } catch (Exception e) { + if (e instanceof HLogLengthMisMatchException) { + throw new HLogLengthMisMatchException( + "Length mismatch while reading HLog."); + } throw new IOException("Cannot get log reader", e); } } Index: src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogLengthMisMatchException.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogLengthMisMatchException.java (revision 0) +++ src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogLengthMisMatchException.java (revision 0) @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.IOException; + +/** + * Exception thrown when the length mismatch happens + * between the one read from DN and the one from NN. + */ +public class HLogLengthMisMatchException extends IOException { + /** + * Create this exception without a message + */ + public HLogLengthMisMatchException() { + super(); + } + + /** + * Create this exception with a message + * + * @param message + * why it failed + */ + public HLogLengthMisMatchException(String message) { + super(message); + } +} Index: src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java (revision 1240082) +++ src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java (working copy) @@ -109,9 +109,15 @@ getFileLength.setAccessible(true); long realLength = ((Long)getFileLength. invoke(realIn, new Object []{})).longValue(); - assert(realLength >= this.length); + if (realLength < this.length) { + throw new HLogLengthMisMatchException( + "Error while trying to get accurate file length while " + + "parsing HLog. It is better to abort the master."); + } adjust = realLength - this.length; - } catch(Exception e) { + } catch (HLogLengthMisMatchException hlme) { + throw hlme; + } catch (Exception e) { SequenceFileLogReader.LOG.warn( "Error while trying to get accurate file length. " + "Truncation / data loss may occur if RegionServers die.", e); Index: src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceLogReader.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceLogReader.java (revision 0) +++ src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceLogReader.java (revision 0) @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +public class InstrumentedSequenceLogReader extends SequenceFileLogReader { + + @Override + public void init(FileSystem fs, Path path, Configuration conf) + throws IOException { + this.conf = conf; + this.path = path; + reader = new InstrumentedWalReader(fs, path, conf); + + } + + static class InstrumentedWalReader extends WALReader { + + InstrumentedWalReader(FileSystem fs, Path p, Configuration c) + throws IOException { + super(fs, p, c); + } + + @Override + protected FSDataInputStream openFile(FileSystem fs, Path file, + int bufferSize, long length) throws IOException { + return new InstrumentedWALReaderFSDataInputStream(super.openFile(fs, + file, bufferSize, length), length); + } + + } + + static class InstrumentedWALReaderFSDataInputStream extends FSDataInputStream { + + InstrumentedWALReaderFSDataInputStream(final FSDataInputStream is, + final long l) throws IOException { + super(is); + } + + @Override + public long getPos() throws IOException { + throw new HLogLengthMisMatchException("Throw exception."); + } + + } +} Index: src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java (revision 1240082) +++ src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java (working copy) @@ -703,7 +703,38 @@ fail("There shouldn't be any exception but: " + e.toString()); } } + + @Test + public void testShouldThrowErrorIfLengthMismatchWhileParsingHLog() + throws Exception { + conf.setBoolean(HBASE_SKIP_ERRORS, false); + Class backupClass = conf.getClass("hbase.regionserver.hlog.reader.impl", + Reader.class); + InstrumentedSequenceFileLogWriter.activateFailure = false; + HLog.resetLogReaderClass(); + try { + conf.setClass("hbase.regionserver.hlog.reader.impl", + InstrumentedSequenceLogReader.class, HLog.Reader.class); + generateHLogs(1, ENTRIES, -1); + fs.initialize(fs.getUri(), conf); + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, hbaseDir, + hlogDir, oldLogDir, fs); + try { + logSplitter.splitLog(); + fail("Should throw error if while parsing we get an error."); + } catch (Exception t) { + assertTrue( + "IOException HLogLengthMisMatchException should be thrown.", + t instanceof IOException); + } + } finally { + conf.setClass("hbase.regionserver.hlog.reader.impl", backupClass, + Reader.class); + HLog.resetLogReaderClass(); + } + } + /** * @throws IOException * @see https://issues.apache.org/jira/browse/HBASE-4862