Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java (revision 1465326) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java (working copy) @@ -20,6 +20,7 @@ import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InterruptedIOException; import java.io.OutputStream; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -126,6 +127,9 @@ private boolean failIfLogDirExists; private WALCoprocessorHost coprocessorHost; + /** attributes for retrying log rolling */ + private final int logRollRetryCount; + private final int pauseTime; private FSDataOutputStream hdfs_out; // FSDataOutputStream associated with the current SequenceFile.writer // Minimum tolerable replicas, if the actual value is lower than it, @@ -338,6 +342,8 @@ this.logrollsize = (long)(this.blocksize * multi); this.optionalFlushInterval = conf.getLong("hbase.regionserver.optionallogflushinterval", 1 * 1000); + this.logRollRetryCount = conf.getInt("hbase.regionserver.logroll.retry.count", 10); + this.pauseTime = conf.getInt("hbase.regionserver.logroll.pause.ms", 1000); this.maxLogs = conf.getInt("hbase.regionserver.maxlogs", 32); this.minTolerableReplication = conf.getInt( @@ -577,18 +583,33 @@ * This method allows subclasses to inject different writers without having to * extend other methods like rollWriter(). * - * @param fs + * @param fileSystem * @param path - * @param conf + * @param configuration * @return Writer instance * @throws IOException */ - protected Writer createWriterInstance(final FileSystem fs, final Path path, - final Configuration conf) throws IOException { + protected Writer createWriterInstance(final FileSystem fileSystem, final Path path, + final Configuration configuration) throws IOException { if (forMeta) { //TODO: set a higher replication for the hlog files (HBASE-6773) } - return HLogFactory.createWriter(fs, path, conf); + IOException lastException = null; + for (int i = 0; i < this.logRollRetryCount; i++) { + try { + return HLogFactory.createWriter(fileSystem, path, configuration); + } catch (IOException ioe) { + lastException = ioe; + try { + Thread.sleep(pauseTime); + } catch (InterruptedException ie) { + IOException iie = new InterruptedIOException(); + iie.initCause(ie); + throw iie; + } + } + } + throw lastException; } /* @@ -721,9 +742,8 @@ private void archiveLogFile(final Path p, final Long seqno) throws IOException { Path newPath = getHLogArchivePath(this.oldLogDir, p); - LOG.info("moving old hlog file " + FSUtils.getPath(p) + - " whose highest sequenceid is " + seqno + " to " + - FSUtils.getPath(newPath)); + LOG.info("moving old hlog file " + FSUtils.getPath(p) + " whose highest sequenceid is " + seqno + + " to " + FSUtils.getPath(newPath)); // Tell our listeners that a log is going to be archived. if (!this.listeners.isEmpty()) { @@ -731,9 +751,22 @@ i.preLogArchive(p, newPath); } } - if (!this.fs.rename(p, newPath)) { - throw new IOException("Unable to rename " + p + " to " + newPath); + boolean result = false; + for (int i = 0; i < this.logRollRetryCount; i++) { + try { + result = this.fs.rename(p, newPath); + break; + } catch (IOException ioe) { + try { + Thread.sleep(this.pauseTime); + } catch (InterruptedException e) { + IOException iie = new InterruptedIOException(); + iie.initCause(e); + throw iie; + } + } } + if (!result) throw new IOException("Unable to rename " + p + " to " + newPath); // Tell our listeners that a log has been archived. if (!this.listeners.isEmpty()) { for (WALActionsListener i : this.listeners) {