From bcf45f9be10c43722dd83f6c920bd7765590c1a9 Mon Sep 17 00:00:00 2001 From: jingyuntian Date: Wed, 7 Nov 2018 11:30:33 +0800 Subject: [PATCH] HBASE-21437 Bypassed procedure throw IllegalArgumentException when its state is WAITING_TIMEOUT --- .../hadoop/hbase/procedure2/ProcedureExecutor.java | 8 ++++++ .../hbase/procedure2/ProcedureTestingUtility.java | 7 +++++ .../hbase/procedure2/TestProcedureBypass.java | 30 ++++++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java index d02ca6e..8bf8fc9 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java @@ -967,6 +967,14 @@ public class ProcedureExecutor { store.update(procedure); } + if (procedure.getState() == ProcedureState.WAITING_TIMEOUT) { + LOG.debug("transform procedure {} from WAITING_TIMEOUT to RUNNABLE", procedure); + if (!procedure.setTimeoutFailure(environment)) { + boolean remove = timeoutExecutor.remove(procedure); + LOG.debug("remove procedure {} from timeoutExecutor {}", procedure, remove); + } + } + // If we don't have the lock, we can't re-submit the queue, // since it is already executing. To get rid of the stuck situation, we // need to restart the master. With the procedure set to bypass, the procedureExecutor diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java index 452e08b..a465bbe 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java @@ -44,6 +44,7 @@ import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.protobuf.ByteString; import org.apache.hbase.thirdparty.com.google.protobuf.BytesValue; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState; public class ProcedureTestingUtility { @@ -423,6 +424,12 @@ public class ProcedureTestingUtility { protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException { } + + @Override + protected synchronized boolean setTimeoutFailure(TEnv env) { + setState(ProcedureProtos.ProcedureState.RUNNABLE); + return false; // 'false' means that this procedure handled the timeout + } } public static class NoopStateMachineProcedure diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureBypass.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureBypass.java index 7d587fd..7fbea8f 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureBypass.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureBypass.java @@ -38,6 +38,7 @@ import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos; @Category({MasterTests.class, SmallTests.class}) @@ -149,6 +150,18 @@ public class TestProcedureBypass { LOG.info("{} finished", proc); } + @Test + public void testBypassingWaitingTimeoutProcedures() throws Exception { + final WaitingTimeoutProcedure proc = new WaitingTimeoutProcedure(); + long id = procExecutor.submitProcedure(proc); + Thread.sleep(500); + // bypass the procedure + assertTrue(procExecutor.bypassProcedure(id, 1000, true, false)); + + htu.waitFor(5000, () -> proc.isSuccess() && proc.isBypass()); + LOG.info("{} finished", proc); + } + @AfterClass public static void tearDown() throws Exception { procExecutor.stop(); @@ -208,6 +221,23 @@ public class TestProcedureBypass { } } + public static class WaitingTimeoutProcedure extends ProcedureTestingUtility.NoopProcedure { + + public WaitingTimeoutProcedure() { + super(); + } + + @Override + protected Procedure[] execute(final TestProcEnv env) + throws ProcedureSuspendedException { + // Always suspend the procedure + setTimeout(50000); + setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT); + skipPersistence(); + throw new ProcedureSuspendedException(); + } + } + public enum StuckStateMachineState { START, THEN, END -- 2.7.4