Index: src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (revision 1465325) +++ src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (working copy) @@ -23,6 +23,7 @@ import java.io.DataOutput; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InterruptedIOException; import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.lang.reflect.InvocationTargetException; @@ -152,6 +153,10 @@ private static Class logWriterClass; private static Class logReaderClass; + /** attributes for retrying log rolling */ + private final int logRollRetryCount; + private final int pauseTime; + private WALCoprocessorHost coprocessorHost; static void resetLogReaderClass() { @@ -423,6 +428,8 @@ if (!fs.exists(oldLogDir) && !HBaseFileSystem.makeDirOnFileSystem(fs, conf, oldLogDir)) { throw new IOException("Unable to mkdir " + this.oldLogDir); } + this.logRollRetryCount = conf.getInt("hbase.regionserver.logroll.retry.count", 10); + this.pauseTime = conf.getInt("hbase.regionserver.logroll.pause.ms", 1000); this.forMeta = forMeta; this.maxLogs = conf.getInt("hbase.regionserver.maxlogs", 32); this.minTolerableReplication = conf.getInt( @@ -701,18 +708,33 @@ * This method allows subclasses to inject different writers without having to * extend other methods like rollWriter(). * - * @param fs + * @param fileSystem * @param path - * @param conf + * @param configuration * @return Writer instance * @throws IOException */ - protected Writer createWriterInstance(final FileSystem fs, final Path path, - final Configuration conf) throws IOException { + protected Writer createWriterInstance(final FileSystem fileSystem, final Path path, + final Configuration configuration) throws IOException { if (forMeta) { //TODO: set a higher replication for the hlog files (HBASE-6773) } - return this.hlogFs.createWriter(fs, conf, path); + IOException lastException = null; + for (int i = 0; i < this.logRollRetryCount; i++) { + try { + return createWriter(fileSystem, path, configuration); + } catch (IOException ioe) { + lastException = ioe; + try { + Thread.sleep(pauseTime); + } catch (InterruptedException ie) { + IOException iie = new InterruptedIOException(); + iie.initCause(ie); + throw iie; + } + } + } + throw lastException; } /** @@ -921,9 +943,8 @@ private void archiveLogFile(final Path p, final Long seqno) throws IOException { Path newPath = getHLogArchivePath(this.oldLogDir, p); - LOG.info("moving old hlog file " + FSUtils.getPath(p) + - " whose highest sequenceid is " + seqno + " to " + - FSUtils.getPath(newPath)); + LOG.info("moving old hlog file " + FSUtils.getPath(p) + " whose highest sequenceid is " + seqno + + " to " + FSUtils.getPath(newPath)); // Tell our listeners that a log is going to be archived. if (!this.listeners.isEmpty()) { @@ -931,9 +952,22 @@ i.preLogArchive(p, newPath); } } - if (!HBaseFileSystem.renameDirForFileSystem(fs, conf, p, newPath)) { - throw new IOException("Unable to rename " + p + " to " + newPath); + boolean result = false; + for (int i = 0; i < this.logRollRetryCount; i++) { + try { + result = this.fs.rename(p, newPath); + break; + } catch (IOException ioe) { + try { + Thread.sleep(this.pauseTime); + } catch (InterruptedException e) { + IOException iie = new InterruptedIOException(); + iie.initCause(e); + throw iie; + } + } } + if (!result) throw new IOException("Unable to rename " + p + " to " + newPath); // Tell our listeners that a log has been archived. if (!this.listeners.isEmpty()) { for (WALActionsListener i : this.listeners) {