From af3d4ec9d2192b430e6628e005e67b684c665fe7 Mon Sep 17 00:00:00 2001 From: zhangduo Date: Thu, 21 Mar 2019 17:42:46 +0800 Subject: [PATCH] HBASE-22074 Should use procedure store to persist the state in reportRegionStateTransition --- .../hbase/shaded/protobuf/ProtobufUtil.java | 32 ++-- .../shaded/protobuf/RequestConverter.java | 28 +-- .../src/main/protobuf/Admin.proto | 3 +- .../src/main/protobuf/MasterProcedure.proto | 3 + .../main/protobuf/RegionServerStatus.proto | 1 + .../master/assignment/AssignProcedure.java | 6 +- .../master/assignment/AssignmentManager.java | 12 +- .../assignment/CloseRegionProcedure.java | 26 ++- .../assignment/OpenRegionProcedure.java | 50 ++++- .../assignment/RegionRemoteProcedureBase.java | 100 +++++++--- .../TransitRegionStateProcedure.java | 175 +++--------------- .../master/assignment/UnassignProcedure.java | 4 +- .../procedure/RSProcedureDispatcher.java | 54 ++---- .../hbase/regionserver/HRegionServer.java | 10 +- .../hbase/regionserver/RSRpcServices.java | 8 +- .../regionserver/RegionServerServices.java | 31 +++- .../hbase/regionserver/SplitRequest.java | 5 +- .../handler/AssignRegionHandler.java | 14 +- .../handler/CloseRegionHandler.java | 3 +- .../handler/OpenRegionHandler.java | 14 +- .../handler/UnassignRegionHandler.java | 15 +- .../procedure/TestServerRemoteProcedure.java | 16 +- 22 files changed, 301 insertions(+), 309 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index 15a8c8a0a3..336c59cdc8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java @@ -3003,28 +3003,32 @@ public final class ProtobufUtil { } /** - * Create a CloseRegionRequest for a given region name - * - * @param regionName the name of the region to close - * @return a CloseRegionRequest - */ - public static CloseRegionRequest buildCloseRegionRequest(ServerName server, - final byte[] regionName) { - return ProtobufUtil.buildCloseRegionRequest(server, regionName, null); - } + * Create a CloseRegionRequest for a given region name + * @param regionName the name of the region to close + * @return a CloseRegionRequest + */ + public static CloseRegionRequest buildCloseRegionRequest(ServerName server, byte[] regionName) { + return ProtobufUtil.buildCloseRegionRequest(server, regionName, null); + } + + public static CloseRegionRequest buildCloseRegionRequest(ServerName server, byte[] regionName, + ServerName destinationServer) { + return buildCloseRegionRequest(server, regionName, destinationServer, -1); + } - public static CloseRegionRequest buildCloseRegionRequest(ServerName server, - final byte[] regionName, ServerName destinationServer) { + public static CloseRegionRequest buildCloseRegionRequest(ServerName server, byte[] regionName, + ServerName destinationServer, long closeProcId) { CloseRegionRequest.Builder builder = CloseRegionRequest.newBuilder(); - RegionSpecifier region = RequestConverter.buildRegionSpecifier( - RegionSpecifierType.REGION_NAME, regionName); + RegionSpecifier region = + RequestConverter.buildRegionSpecifier(RegionSpecifierType.REGION_NAME, regionName); builder.setRegion(region); - if (destinationServer != null){ + if (destinationServer != null) { builder.setDestinationServer(toServerName(destinationServer)); } if (server != null) { builder.setServerStartCode(server.getStartcode()); } + builder.setCloseProcId(closeProcId); return builder.build(); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java index 36c8fab23a..0c58d4b2f1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java @@ -940,27 +940,6 @@ public final class RequestConverter { return builder.build(); } - /** - * Create a protocol buffer OpenRegionRequest to open a list of regions - * @param server the serverName for the RPC - * @param regionOpenInfos info of a list of regions to open - * @return a protocol buffer OpenRegionRequest - */ - public static OpenRegionRequest buildOpenRegionRequest(ServerName server, - final List>> regionOpenInfos) { - OpenRegionRequest.Builder builder = OpenRegionRequest.newBuilder(); - for (Pair> regionOpenInfo : regionOpenInfos) { - builder.addOpenInfo(buildRegionOpenInfo(regionOpenInfo.getFirst(), - regionOpenInfo.getSecond())); - } - if (server != null) { - builder.setServerStartCode(server.getStartcode()); - } - // send the master's wall clock time as well, so that the RS can refer to it - builder.setMasterSystemTime(EnvironmentEdgeManager.currentTime()); - return builder.build(); - } - /** * Create a protocol buffer OpenRegionRequest for a given region * @param server the serverName for the RPC @@ -971,7 +950,7 @@ public final class RequestConverter { public static OpenRegionRequest buildOpenRegionRequest(ServerName server, final RegionInfo region, List favoredNodes) { OpenRegionRequest.Builder builder = OpenRegionRequest.newBuilder(); - builder.addOpenInfo(buildRegionOpenInfo(region, favoredNodes)); + builder.addOpenInfo(buildRegionOpenInfo(region, favoredNodes, -1L)); if (server != null) { builder.setServerStartCode(server.getStartcode()); } @@ -1622,8 +1601,8 @@ public final class RequestConverter { /** * Create a RegionOpenInfo based on given region info and version of offline node */ - public static RegionOpenInfo buildRegionOpenInfo( - final RegionInfo region, final List favoredNodes) { + public static RegionOpenInfo buildRegionOpenInfo(RegionInfo region, List favoredNodes, + long openProcId) { RegionOpenInfo.Builder builder = RegionOpenInfo.newBuilder(); builder.setRegion(ProtobufUtil.toRegionInfo(region)); if (favoredNodes != null) { @@ -1631,6 +1610,7 @@ public final class RequestConverter { builder.addFavoredNodes(ProtobufUtil.toServerName(server)); } } + builder.setOpenProcId(openProcId); return builder.build(); } diff --git a/hbase-protocol-shaded/src/main/protobuf/Admin.proto b/hbase-protocol-shaded/src/main/protobuf/Admin.proto index c622d589c6..85b9113a27 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Admin.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Admin.proto @@ -88,6 +88,7 @@ message OpenRegionRequest { repeated ServerName favored_nodes = 3; // open region for distributedLogReplay // optional bool DEPRECATED_openForDistributedLogReplay = 4; + optional int64 open_proc_id = 5 [default = -1]; } } @@ -102,7 +103,6 @@ message OpenRegionResponse { } message WarmupRegionRequest { - required RegionInfo regionInfo = 1; } @@ -120,6 +120,7 @@ message CloseRegionRequest { optional ServerName destination_server = 4; // the intended server for this RPC. optional uint64 serverStartCode = 5; + optional int64 close_proc_id = 6 [default = -1]; } message CloseRegionResponse { diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto index 64ac39802d..39878d41f9 100644 --- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto +++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto @@ -28,6 +28,7 @@ import "HBase.proto"; import "RPC.proto"; import "Snapshot.proto"; import "Replication.proto"; +import "RegionServerStatus.proto"; // ============================================================================ // WARNING - Compatibility rules @@ -551,6 +552,8 @@ message RegionStateTransitionStateData { message RegionRemoteProcedureBaseStateData { required RegionInfo region = 1; required ServerName target_server = 2; + optional RegionStateTransition.TransitionCode transition_code = 3; + optional int64 seq_id = 4; } message OpenRegionProcedureStateData { diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto index 002432a2f2..0137cb1608 100644 --- a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto +++ b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto @@ -96,6 +96,7 @@ message RegionStateTransition { /** For newly opened region, the open seq num is needed */ optional uint64 open_seq_num = 3; + repeated int64 proc_id = 4; enum TransitionCode { OPENED = 0; FAILED_OPEN = 1; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java index 33a35453e0..35510d6eef 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java @@ -21,7 +21,6 @@ import java.io.IOException; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.exceptions.UnexpectedStateException; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; -import org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher.RegionOpenOperation; import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; @@ -38,7 +37,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProto * @deprecated Do not use any more. * @see TransitRegionStateProcedure */ -// TODO: Add being able to assign a region to open read-only. @Deprecated @InterfaceAudience.Private public class AssignProcedure extends RegionTransitionProcedure { @@ -121,9 +119,7 @@ public class AssignProcedure extends RegionTransitionProcedure { @Override public RemoteOperation remoteCallBuild(final MasterProcedureEnv env, final ServerName serverName) { - assert serverName.equals(getRegionState(env).getRegionLocation()); - return new RegionOpenOperation(this, getRegionInfo(), - env.getAssignmentManager().getFavoredNodes(getRegionInfo()), false); + return null; } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index 2d0c3bee9f..5e43637bd8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -835,8 +835,10 @@ public class AssignmentManager { case CLOSED: assert transition.getRegionInfoCount() == 1 : transition; final RegionInfo hri = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0)); + long procId = + transition.getProcIdCount() > 0 ? transition.getProcId(0) : Procedure.NO_PROC_ID; updateRegionTransition(serverName, transition.getTransitionCode(), hri, - transition.hasOpenSeqNum() ? transition.getOpenSeqNum() : HConstants.NO_SEQNUM); + transition.hasOpenSeqNum() ? transition.getOpenSeqNum() : HConstants.NO_SEQNUM, procId); break; case READY_TO_SPLIT: case SPLIT: @@ -903,7 +905,7 @@ public class AssignmentManager { } private void updateRegionTransition(ServerName serverName, TransitionCode state, - RegionInfo regionInfo, long seqId) throws IOException { + RegionInfo regionInfo, long seqId, long procId) throws IOException { checkMetaLoaded(regionInfo); RegionStateNode regionNode = regionStates.getRegionStateNode(regionInfo); @@ -919,7 +921,7 @@ public class AssignmentManager { ServerStateNode serverNode = regionStates.getOrCreateServer(serverName); regionNode.lock(); try { - if (!reportTransition(regionNode, serverNode, state, seqId)) { + if (!reportTransition(regionNode, serverNode, state, seqId, procId)) { // Don't log WARN if shutting down cluster; during shutdown. Avoid the below messages: // 2018-08-13 10:45:10,551 WARN ...AssignmentManager: No matching procedure found for // rit=OPEN, location=ve0538.halxg.cloudera.com,16020,1533493000958, @@ -941,14 +943,14 @@ public class AssignmentManager { } private boolean reportTransition(RegionStateNode regionNode, ServerStateNode serverNode, - TransitionCode state, long seqId) throws IOException { + TransitionCode state, long seqId, long procId) throws IOException { ServerName serverName = serverNode.getServerName(); TransitRegionStateProcedure proc = regionNode.getProcedure(); if (proc == null) { return false; } proc.reportTransition(master.getMasterProcedureExecutor().getEnvironment(), regionNode, - serverName, state, seqId); + serverName, state, seqId, procId); return true; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/CloseRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/CloseRegionProcedure.java index f867e96459..4fd60f0ead 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/CloseRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/CloseRegionProcedure.java @@ -20,15 +20,17 @@ package org.apache.hadoop.hbase.master.assignment; import java.io.IOException; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.exceptions.UnexpectedStateException; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher.RegionCloseOperation; import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteOperation; import org.apache.yetus.audience.InterfaceAudience; + import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.CloseRegionProcedureStateData; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; /** * The remote procedure used to close a region. @@ -46,9 +48,9 @@ public class CloseRegionProcedure extends RegionRemoteProcedureBase { super(); } - public CloseRegionProcedure(RegionInfo region, ServerName targetServer, - ServerName assignCandidate) { - super(region, targetServer); + public CloseRegionProcedure(TransitRegionStateProcedure parent, RegionInfo region, + ServerName targetServer, ServerName assignCandidate) { + super(parent, region, targetServer); this.assignCandidate = assignCandidate; } @@ -59,7 +61,7 @@ public class CloseRegionProcedure extends RegionRemoteProcedureBase { @Override public RemoteOperation remoteCallBuild(MasterProcedureEnv env, ServerName remote) { - return new RegionCloseOperation(this, region, assignCandidate); + return new RegionCloseOperation(this, region, getProcId(), assignCandidate); } @Override @@ -88,7 +90,17 @@ public class CloseRegionProcedure extends RegionRemoteProcedureBase { } @Override - protected boolean shouldDispatch(RegionStateNode regionNode) { - return regionNode.isInState(RegionState.State.CLOSING); + protected void reportTransition(RegionStateNode regionNode, TransitionCode transitionCode, + long seqId) throws IOException { + if (transitionCode != TransitionCode.CLOSED) { + throw new UnexpectedStateException("Received report unexpected " + transitionCode + + " transition, " + regionNode.toShortString() + ", " + this + ", expected CLOSED."); + } + } + + @Override + protected void updateTransition(MasterProcedureEnv env, RegionStateNode regionNode, + TransitionCode transitionCode, long seqId) throws IOException { + env.getAssignmentManager().regionClosed(regionNode, true); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/OpenRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/OpenRegionProcedure.java index 4b3a976f28..fb7fdf662a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/OpenRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/OpenRegionProcedure.java @@ -20,15 +20,18 @@ package org.apache.hadoop.hbase.master.assignment; import java.io.IOException; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.exceptions.UnexpectedStateException; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher.RegionOpenOperation; import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteOperation; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.OpenRegionProcedureStateData; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; /** * The remote procedure used to open a region. @@ -36,12 +39,15 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.O @InterfaceAudience.Private public class OpenRegionProcedure extends RegionRemoteProcedureBase { + private static final Logger LOG = LoggerFactory.getLogger(OpenRegionProcedure.class); + public OpenRegionProcedure() { super(); } - public OpenRegionProcedure(RegionInfo region, ServerName targetServer) { - super(region, targetServer); + public OpenRegionProcedure(TransitRegionStateProcedure parent, RegionInfo region, + ServerName targetServer) { + super(parent, region, targetServer); } @Override @@ -51,8 +57,7 @@ public class OpenRegionProcedure extends RegionRemoteProcedureBase { @Override public RemoteOperation remoteCallBuild(MasterProcedureEnv env, ServerName remote) { - return new RegionOpenOperation(this, region, env.getAssignmentManager().getFavoredNodes(region), - false); + return new RegionOpenOperation(this, region, getProcId()); } @Override @@ -73,7 +78,38 @@ public class OpenRegionProcedure extends RegionRemoteProcedureBase { } @Override - protected boolean shouldDispatch(RegionStateNode regionNode) { - return regionNode.isInState(RegionState.State.OPENING); + protected void reportTransition(RegionStateNode regionNode, TransitionCode transitionCode, + long seqId) throws IOException { + switch (transitionCode) { + case OPENED: + // this is the openSeqNum + if (seqId < 0) { + throw new UnexpectedStateException("Received report unexpected " + TransitionCode.OPENED + + " transition openSeqNum=" + seqId + ", " + regionNode + ", proc=" + this); + } + break; + case FAILED_OPEN: + // nothing to check + break; + default: + throw new UnexpectedStateException( + "Received report unexpected " + transitionCode + " transition, " + + regionNode.toShortString() + ", " + this + ", expected OPENED or FAILED_OPEN."); + } + } + + @Override + protected void updateTransition(MasterProcedureEnv env, RegionStateNode regionNode, + TransitionCode transitionCode, long openSeqNum) throws IOException { + if (openSeqNum < regionNode.getOpenSeqNum()) { + LOG.warn( + "Received report {} transition from {} for {}, pid={} but the new openSeqNum {}" + + " is less than the current one {}, ignoring...", + TransitionCode.OPENED, targetServer, regionNode, getProcId(), openSeqNum, + regionNode.getOpenSeqNum()); + } else { + regionNode.setOpenSeqNum(openSeqNum); + } + env.getAssignmentManager().regionOpened(regionNode); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java index f6d3a2eaa5..65f01e2f78 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java @@ -18,9 +18,11 @@ package org.apache.hadoop.hbase.master.assignment; import java.io.IOException; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.exceptions.UnexpectedStateException; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.TableProcedureInterface; import org.apache.hadoop.hbase.procedure2.FailedRemoteDispatchException; @@ -37,6 +39,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionRemoteProcedureBaseStateData; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; /** * The base class for the remote procedures used to open/close a region. @@ -53,16 +56,20 @@ public abstract class RegionRemoteProcedureBase extends Procedure - * This could happen when master restarts. Since we do not know whether a request has already been - * sent to the region server after we add a remote operation to the dispatcher, so the safe way is - * to not persist the dispatched field and try to add the remote operation again. But it is - * possible that we do have already sent the request to region server and it has also sent back - * the response, so here we need to check the region state, if it is not in the expecting state, - * we should give up, otherwise we may hang for ever, as the region server will just ignore - * redundant calls. - */ - protected abstract boolean shouldDispatch(RegionStateNode regionNode); + // do some checks to see if the report is valid, without actually updating meta. + protected abstract void reportTransition(RegionStateNode regionNode, + TransitionCode transitionCode, long seqId) throws IOException; + + // should be called with RegionStateNode locked, to avoid race with the execute method below + public void reportTransition(MasterProcedureEnv env, RegionStateNode regionNode, + ServerName serverName, TransitionCode transitionCode, long seqId) throws IOException { + if (this.transitionCode != null) { + // should be a retry. + return; + } + if (!targetServer.equals(serverName)) { + throw new UnexpectedStateException("Received report from " + serverName + ", expected " + + targetServer + ", " + regionNode + ", proc=" + this); + } + this.transitionCode = transitionCode; + this.seqId = seqId; + boolean succ = false; + try { + // Persist the transition code and openSeqNum(if provided). + // We should not update the hbase:meta directly as this may cause races when master restarts, + // as the old active master may incorrectly report back to RS and cause the new master to hang + // on a OpenRegionProcedure forever. See HBASE-22060 and HBASE-22074 for more details. + env.getMasterServices().getMasterProcedureExecutor().getStore().update(this); + regionNode.getProcedureEvent().wake(env.getProcedureScheduler()); + succ = true; + } finally { + if (!succ) { + // unset the state + this.transitionCode = null; + this.seqId = HConstants.NO_SEQNUM; + } + } + } + + private TransitRegionStateProcedure getParent(MasterProcedureEnv env) { + return (TransitRegionStateProcedure) env.getMasterServices().getMasterProcedureExecutor() + .getProcedure(getParentProcId()); + } + + // actually update the state to meta + protected abstract void updateTransition(MasterProcedureEnv env, RegionStateNode regionNode, + TransitionCode transitionCode, long seqId) throws IOException; @Override protected Procedure[] execute(MasterProcedureEnv env) throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException { - if (dispatched) { - // we are done, the parent procedure will check whether we are succeeded. - return null; - } RegionStateNode regionNode = getRegionNode(env); regionNode.lock(); try { - if (!shouldDispatch(regionNode)) { + if (transitionCode != null) { + // TODO: this could fail, and we need to implement the retry with interval logic. + updateTransition(env, regionNode, transitionCode, seqId); + getParent(env).unattachRemoteProc(this); return null; } // The code which wakes us up also needs to lock the RSN so here we do not need to synchronize @@ -170,10 +207,13 @@ public abstract class RegionRemoteProcedureBase extends Procedure future = submitProcedure(openRegionProcedure); Thread.sleep(2000); rsDispatcher.removeNode(worker); try { future.get(2000, TimeUnit.MILLISECONDS); + fail(); } catch (TimeoutException e) { LOG.info("timeout is expected"); } -- 2.17.1