diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
index 79ff1bc..9ea60ad 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.exceptions.TimeoutIOException;
import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
@@ -164,6 +165,8 @@ public class FSHLog implements WAL {
private static final int DEFAULT_SLOW_SYNC_TIME_MS = 100; // in ms
+ private static final int DEFAULT_WAL_SYNC_TIMEOUT_MS = 5 * 60 * 1000; // in ms, 5min
+
/**
* The nexus at which all incoming handlers meet. Does appends and sync with an ordering.
* Appends and syncs are each put on the ring which means handlers need to
@@ -281,6 +284,8 @@ public class FSHLog implements WAL {
private final int slowSyncNs;
+ private final long walSyncTimeout;
+
// If live datanode count is lower than the default replicas value,
// RollWriter will be triggered in each sync(So the RollWriter will be
// triggered one by one in a short time). Using it as a workaround to slow
@@ -535,6 +540,8 @@ public class FSHLog implements WAL {
this.slowSyncNs =
1000000 * conf.getInt("hbase.regionserver.hlog.slowsync.ms",
DEFAULT_SLOW_SYNC_TIME_MS);
+ this.walSyncTimeout = conf.getLong("hbase.regionserver.hlog.sync.timeout",
+ DEFAULT_WAL_SYNC_TIMEOUT_MS);
// This is the 'writer' -- a single threaded executor. This single thread 'consumes' what is
// put on the ring buffer.
@@ -1380,8 +1387,14 @@ public class FSHLog implements WAL {
private Span blockOnSync(final SyncFuture syncFuture) throws IOException {
// Now we have published the ringbuffer, halt the current thread until we get an answer back.
try {
- syncFuture.get();
+ syncFuture.get(walSyncTimeout);
return syncFuture.getSpan();
+ } catch (TimeoutIOException tioe) {
+ // SyncFuture reuse by thread, if TimeoutIOException happens, ringbuffer
+ // still refer to it, so if this thread use it next time may get a wrong
+ // result.
+ this.syncFuturesByHandler.remove(Thread.currentThread());
+ throw tioe;
} catch (InterruptedException ie) {
LOG.warn("Interrupted", ie);
throw convertInterruptedExceptionToIOException(ie);
@@ -1774,6 +1787,12 @@ public class FSHLog implements WAL {
} catch (Exception e) {
// Failed append. Record the exception.
this.exception = e;
+ // invoking cleanupOutstandingSyncsOnException when append failed with exception,
+ // it will cleanup existing sync requests recorded in syncFutures but not offered to SyncRunner yet,
+ // so there won't be any sync future left over if no further truck published to disruptor.
+ cleanupOutstandingSyncsOnException(sequence,
+ this.exception instanceof DamagedWALException ? this.exception
+ : new DamagedWALException("On sync", this.exception));
// Return to keep processing events coming off the ringbuffer
return;
} finally {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SyncFuture.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SyncFuture.java
index 7de8367..0720de8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SyncFuture.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SyncFuture.java
@@ -21,6 +21,8 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.exceptions.TimeoutIOException;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.htrace.Span;
/**
@@ -105,6 +107,7 @@ class SyncFuture {
this.doneSequence = NOT_DONE;
this.ringBufferSequence = sequence;
this.span = span;
+ this.throwable = null;
return this;
}
@@ -162,9 +165,16 @@ class SyncFuture {
throw new UnsupportedOperationException();
}
- public synchronized long get() throws InterruptedException, ExecutionException {
+ public synchronized long get(long timeout) throws InterruptedException,
+ ExecutionException, TimeoutIOException {
+ final long done = EnvironmentEdgeManager.currentTime() + timeout;
while (!isDone()) {
wait(1000);
+ if (EnvironmentEdgeManager.currentTime() >= done) {
+ throw new TimeoutIOException("Failed to get sync result after "
+ + timeout + " ms for ringBufferSequence=" + this.ringBufferSequence
+ + ", WAL system stuck?");
+ }
}
if (this.throwable != null) throw new ExecutionException(this.throwable);
return this.doneSequence;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
index 457d859..cf8b5de 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
@@ -20,11 +20,10 @@ package org.apache.hadoop.hbase.regionserver.wal;
import java.io.IOException;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
-
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.wal.WALKey;
/**
@@ -93,9 +92,8 @@ public interface WALActionsListener {
* It only exists to get scope when replicating. Scope should be in the WALKey and not need
* us passing in a htd.
*/
- void visitLogEntryBeforeWrite(
- HTableDescriptor htd, WALKey logKey, WALEdit logEdit
- );
+ void visitLogEntryBeforeWrite(HTableDescriptor htd, WALKey logKey,
+ WALEdit logEdit) throws IOException;
/**
* For notification post append to the writer. Used by metrics system at least.
@@ -136,7 +134,9 @@ public interface WALActionsListener {
public void visitLogEntryBeforeWrite(HRegionInfo info, WALKey logKey, WALEdit logEdit) {}
@Override
- public void visitLogEntryBeforeWrite(HTableDescriptor htd, WALKey logKey, WALEdit logEdit) {}
+ public void visitLogEntryBeforeWrite(HTableDescriptor htd, WALKey logKey,
+ WALEdit logEdit) throws IOException {
+ }
@Override
public void postAppend(final long entryLen, final long elapsedTimeMillis) {}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
index 56c6c60..d82d1df 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
@@ -18,7 +18,6 @@
*/
package org.apache.hadoop.hbase.regionserver;
-
import static org.junit.Assert.assertTrue;
import java.io.IOException;
@@ -31,14 +30,21 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CellScanner;
+import org.apache.hadoop.hbase.ChoreService;
+import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.regionserver.wal.DamagedWALException;
import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
+import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
+import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -47,6 +53,8 @@ import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.wal.WALKey;
import org.apache.hadoop.hbase.wal.WALProvider.Writer;
+import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
@@ -282,6 +290,281 @@ public class TestWALLockup {
}
/**
+ * Reproduce locking up that happens when there's no further syncs after
+ * append fails, and causing an isolated sync then infinite wait. See
+ * HBASE-16960. If below is broken, we will see this test timeout because it
+ * is locked up.
+ *