From 3bb248d380134c8857e637b5a2a13a50f49a5e09 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Wed, 17 Jan 2018 22:35:35 -0800 Subject: [PATCH] HBASE-19815 Flakey TestAssignmentManager.testAssignWithRandExec Part Two. Add the ServerCrashProcedure#handleRIT behavior to RecoverMetaProcedure. Add debug in the test. --- .../master/procedure/RecoverMetaProcedure.java | 26 +++++++++++++++++++--- .../master/assignment/TestAssignmentManager.java | 9 +++++++- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java index 50ef3e0da2..bb8732eabc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java @@ -28,6 +28,8 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.assignment.AssignProcedure; +import org.apache.hadoop.hbase.master.assignment.AssignmentManager; +import org.apache.hadoop.hbase.master.assignment.RegionTransitionProcedure; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; import org.apache.hadoop.hbase.procedure2.ProcedureYieldException; @@ -126,17 +128,35 @@ public class RecoverMetaProcedure RegionInfoBuilder.FIRST_META_REGIONINFO, this.replicaId); AssignProcedure metaAssignProcedure; + AssignmentManager am = master.getAssignmentManager(); if (failedMetaServer != null) { + // Is the region stuck assigning to this failedMetaServer? If so, cancel the call + // just as we do over in ServerCrashProcedure#handleRIT except less to do here. + RegionTransitionProcedure rtp = am.getRegionStates().getRegionTransitionProcedure(hri); + if (rtp != null) { + // Make sure the RIT is against this crashed server. In the case where there are many + // processings of a crashed server -- backed up for whatever reason (slow WAL split) + // -- then a previous SCP may have already failed an assign, etc., and it may have a + // new location target; DO NOT fail these else we make for assign flux. + ServerName rtpServerName = rtp.getServer(env); + if (rtpServerName == null) { + LOG.warn("RIT with ServerName null! " + rtp); + } else if (rtpServerName.equals(failedMetaServer)) { + LOG.info("pid=" + getProcId() + " found RIT " + rtp + "; " + + rtp.getRegionState(env).toShortString()); + rtp.remoteCallFailed(env, failedMetaServer, + new ServerCrashException(getProcId(), failedMetaServer)); + } + } LOG.info(this + "; Assigning meta with new plan. previous meta server=" + failedMetaServer); - metaAssignProcedure = master.getAssignmentManager().createAssignProcedure(hri); + metaAssignProcedure = am.createAssignProcedure(hri); } else { // get server carrying meta from zk ServerName metaServer = MetaTableLocator.getMetaRegionState(master.getZooKeeper()).getServerName(); LOG.info(this + "; Retaining meta assignment to server=" + metaServer); - metaAssignProcedure = - master.getAssignmentManager().createAssignProcedure(hri, metaServer); + metaAssignProcedure = am.createAssignProcedure(hri, metaServer); } addChildProcedure(metaAssignProcedure); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java index 3ab915b998..9b9f624238 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; +import java.io.InterruptedIOException; import java.net.SocketTimeoutException; import java.util.NavigableMap; import java.util.Random; @@ -206,7 +207,7 @@ public class TestAssignmentManager { rsDispatcher.setMockRsExecutor(new RandRsExecutor()); // Loop a bunch of times so we hit various combos of exceptions. for (int i = 0; i < 10; i++) { - LOG.info("" + i); + LOG.info("ROUND=" + i); AssignProcedure proc = am.createAssignProcedure(hri); waitOnFuture(submitProcedure(proc)); } @@ -445,6 +446,12 @@ public class TestAssignmentManager { return future.get(5, TimeUnit.SECONDS); } catch (ExecutionException e) { LOG.info("ExecutionException", e); + Exception ee = (Exception)e.getCause(); + if (ee instanceof InterruptedIOException) { + for (Procedure p: this.master.getMasterProcedureExecutor().getProcedures()) { + LOG.info(p.toStringDetails()); + } + } throw (Exception)e.getCause(); } } -- 2.11.0 (Apple Git-81)