From 10de5b77dfc46877c1222cf70e5463a7c072b377 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Thu, 20 Sep 2018 16:53:58 -0700 Subject: [PATCH] HBASE-21213 [hbck2] bypass leaves behind state in RegionStates when assign/unassign bypass on an assign/unassign leaves region in RIT and the RegionStateNode loaded with the bypassed procedure. First implementation had assign/unassign cleanup leftover state. Second implementation keeps the state in place as a fence against other Procedures and instead adds a 'force' function that hbck2 can set on assigns/unassigns to override the fencing. M hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java Have bypass take an environment like all other methods so subclasses. Fix javadoc issues. M hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java Javadoc issues. Pass environment when we invoke bypass. M hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Rename waitUntilNamespace... etc. to align with how these method types are named elsehwere .. i.e. waitFor rather than waitUntil.. M hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java Cleanup message we emit when we find an exisitng procedure working against this entity. Add support for a force function which allows Assigns/Unassigns force ownership of the Region entity. A hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionBypass.java Test bypass and force. M hbase-shell/src/main/ruby/shell/commands/list_procedures.rb Minor cleanup of the json output... do iso8601 timestamps. --- .../org/apache/hadoop/hbase/client/HBaseHbck.java | 10 +- .../java/org/apache/hadoop/hbase/client/Hbck.java | 27 +++- .../hbase/shaded/protobuf/RequestConverter.java | 10 +- .../apache/hadoop/hbase/procedure2/Procedure.java | 29 ++-- .../hadoop/hbase/procedure2/ProcedureExecutor.java | 13 +- .../hadoop/hbase/procedure2/ProcedureUtil.java | 2 +- .../hbase/procedure2/RemoteProcedureException.java | 6 +- .../hbase/procedure2/TestProcedureBypass.java | 3 - .../src/main/protobuf/Master.proto | 5 +- .../src/main/protobuf/MasterProcedure.proto | 1 + .../hbase/rsgroup/RSGroupInfoManagerImpl.java | 3 +- .../org/apache/hadoop/hbase/master/HMaster.java | 8 +- .../hadoop/hbase/master/MasterRpcServices.java | 19 ++- .../hbase/master/assignment/AssignProcedure.java | 13 +- .../hbase/master/assignment/AssignmentManager.java | 27 +++- .../assignment/RegionTransitionProcedure.java | 68 ++++++-- .../hbase/master/assignment/UnassignProcedure.java | 11 +- .../master/procedure/MasterProcedureUtil.java | 16 ++ .../hbase/master/procedure/ProcedureDescriber.java | 3 +- .../master/procedure/ProcedurePrepareLatch.java | 2 +- .../hbase/master/procedure/ProcedureSyncWait.java | 10 +- .../apache/hadoop/hbase/TestMetaTableAccessor.java | 4 +- .../hbase/master/assignment/TestRegionBypass.java | 179 +++++++++++++++++++++ .../main/ruby/shell/commands/list_procedures.rb | 7 +- 24 files changed, 390 insertions(+), 86 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionBypass.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseHbck.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseHbck.java index a8daa7b5db..9d594695d6 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseHbck.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseHbck.java @@ -102,11 +102,12 @@ public class HBaseHbck implements Hbck { } @Override - public List assigns(List encodedRegionNames) throws IOException { + public List assigns(List encodedRegionNames, boolean force) + throws IOException { try { MasterProtos.AssignsResponse response = this.hbck.assigns(rpcControllerFactory.newController(), - RequestConverter.toAssignRegionsRequest(encodedRegionNames)); + RequestConverter.toAssignRegionsRequest(encodedRegionNames, force)); return response.getPidList(); } catch (ServiceException se) { LOG.debug(toCommaDelimitedString(encodedRegionNames), se); @@ -115,11 +116,12 @@ public class HBaseHbck implements Hbck { } @Override - public List unassigns(List encodedRegionNames) throws IOException { + public List unassigns(List encodedRegionNames, boolean force) + throws IOException { try { MasterProtos.UnassignsResponse response = this.hbck.unassigns(rpcControllerFactory.newController(), - RequestConverter.toUnassignRegionsRequest(encodedRegionNames)); + RequestConverter.toUnassignRegionsRequest(encodedRegionNames, force)); return response.getPidList(); } catch (ServiceException se) { LOG.debug(toCommaDelimitedString(encodedRegionNames), se); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Hbck.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Hbck.java index 5c9a862feb..b40286a657 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Hbck.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Hbck.java @@ -28,11 +28,14 @@ import org.apache.yetus.audience.InterfaceAudience; /** * Hbck fixup tool APIs. Obtain an instance from {@link ClusterConnection#getHbck()} and call * {@link #close()} when done. - *

WARNING: the below methods can damage the cluster. For experienced users only. + *

WARNING: the below methods can damage the cluster. It may leave the cluster in an + * indeterminate state, e.g. region not assigned, or some hdfs files left behind. After running + * any of the below, operators may have to do some clean up on hdfs or schedule some assign + * procedures to get regions back online. DO AT YOUR OWN RISK. For experienced users only. * * @see ConnectionFactory * @see ClusterConnection - * @since 2.2.0 + * @since 2.0.2, 2.1.1 */ @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.HBCK) public interface Hbck extends Abortable, Closeable { @@ -49,22 +52,38 @@ public interface Hbck extends Abortable, Closeable { * -- good if many Regions to online -- and it will schedule the assigns even in the case where * Master is initializing (as long as the ProcedureExecutor is up). Does NOT call Coprocessor * hooks. + * @param force You need to add the force for the case where a region has previously been + * bypassed. When a Procedure has been bypassed, a Procedure will have completed + * but no other Procedure will be able to make progress on the target entity + * (intentionally). This force flag will override this fencing mechanism. * @param encodedRegionNames Region encoded names; e.g. 1588230740 is the hard-coded encoding * for hbase:meta region and de00010733901a05f5a2a3a382e27dd4 is an * example of what a random user-space encoded Region name looks like. */ - List assigns(List encodedRegionNames) throws IOException; + List assigns(List encodedRegionNames, boolean force) throws IOException; + + default List assigns(List encodedRegionNames) throws IOException { + return assigns(encodedRegionNames, false); + } /** * Like {@link Admin#unassign(byte[], boolean)} but 'raw' in that it can do more than one Region * at a time -- good if many Regions to offline -- and it will schedule the assigns even in the * case where Master is initializing (as long as the ProcedureExecutor is up). Does NOT call * Coprocessor hooks. + * @param force You need to add the force for the case where a region has previously been + * bypassed. When a Procedure has been bypassed, a Procedure will have completed + * but no other Procedure will be able to make progress on the target entity + * (intentionally). This force flag will override this fencing mechanism. * @param encodedRegionNames Region encoded names; e.g. 1588230740 is the hard-coded encoding * for hbase:meta region and de00010733901a05f5a2a3a382e27dd4 is an * example of what a random user-space encoded Region name looks like. */ - List unassigns(List encodedRegionNames) throws IOException; + List unassigns(List encodedRegionNames, boolean force) throws IOException; + + default List unassigns(List encodedRegionNames) throws IOException { + return unassigns(encodedRegionNames, false); + } /** * Bypass specified procedure and move it to completion. Procedure is marked completed but diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java index 6ab81e22f2..f5875607d6 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java @@ -1883,16 +1883,18 @@ public final class RequestConverter { // HBCK2 public static MasterProtos.AssignsRequest toAssignRegionsRequest( - List encodedRegionNames) { + List encodedRegionNames, boolean force) { MasterProtos.AssignsRequest.Builder b = MasterProtos.AssignsRequest.newBuilder(); - return b.addAllRegion(toEncodedRegionNameRegionSpecifiers(encodedRegionNames)).build(); + return b.addAllRegion(toEncodedRegionNameRegionSpecifiers(encodedRegionNames)). + setForce(force).build(); } public static MasterProtos.UnassignsRequest toUnassignRegionsRequest( - List encodedRegionNames) { + List encodedRegionNames, boolean force) { MasterProtos.UnassignsRequest.Builder b = MasterProtos.UnassignsRequest.newBuilder(); - return b.addAllRegion(toEncodedRegionNameRegionSpecifiers(encodedRegionNames)).build(); + return b.addAllRegion(toEncodedRegionNameRegionSpecifiers(encodedRegionNames)). + setForce(force).build(); } private static List toEncodedRegionNameRegionSpecifiers( diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java index a832c783fe..6d87e77464 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java @@ -145,12 +145,16 @@ public abstract class Procedure implements Comparable

Bypassing a procedure is not like aborting. Aborting a procedure will trigger * a rollback. And since the {@link #abort(Object)} method is overrideable * Some procedures may have chosen to ignore the aborting. */ @@ -161,12 +165,15 @@ public abstract class Procedure implements Comparable implements Comparable { private Configuration conf; /** - * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing + * Created in the {@link #init(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing * resource handling rather than observing in a #join is unexpected). * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery * (Should be ok). @@ -316,7 +316,7 @@ public class ProcedureExecutor { private ThreadGroup threadGroup; /** - * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing + * Created in the {@link #init(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing * resource handling rather than observing in a #join is unexpected). * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery * (Should be ok). @@ -324,7 +324,7 @@ public class ProcedureExecutor { private CopyOnWriteArrayList workerThreads; /** - * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing + * Created in the {@link #init(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing * resource handling rather than observing in a #join is unexpected). * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery * (Should be ok). @@ -968,7 +968,7 @@ public class ProcedureExecutor { * Bypass a procedure. If the procedure is set to bypass, all the logic in * execute/rollback will be ignored and it will return success, whatever. * It is used to recover buggy stuck procedures, releasing the lock resources - * and letting other procedures to run. Bypassing one procedure (and its ancestors will + * and letting other procedures run. Bypassing one procedure (and its ancestors will * be bypassed automatically) may leave the cluster in a middle state, e.g. region * not assigned, or some hdfs files left behind. After getting rid of those stuck procedures, * the operators may have to do some clean up on hdfs or schedule some assign procedures @@ -1010,7 +1010,7 @@ public class ProcedureExecutor { boolean bypassProcedure(long pid, long lockWait, boolean force) throws IOException { Procedure procedure = getProcedure(pid); if (procedure == null) { - LOG.debug("Procedure with id={} does not exist, skipping bypass", pid); + LOG.debug("Procedure pid={} does not exist, skipping bypass", pid); return false; } @@ -1043,6 +1043,7 @@ public class ProcedureExecutor { LOG.debug("Bypassing procedures in RUNNABLE, WAITING and WAITING_TIMEOUT states " + "(with no parent), {}", procedure); + // Question: how is the bypass done here? return false; } @@ -1052,7 +1053,7 @@ public class ProcedureExecutor { Procedure current = procedure; while (current != null) { LOG.debug("Bypassing {}", current); - current.bypass(); + current.bypass(getEnvironment()); store.update(procedure); long parentID = current.getParentProcId(); current = getProcedure(parentID); diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureUtil.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureUtil.java index 8c8746e84b..e64db6e7a6 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureUtil.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureUtil.java @@ -271,7 +271,7 @@ public final class ProcedureUtil { } if (proto.getBypass()) { - proc.bypass(); + proc.bypass(null); } ProcedureStateSerializer serializer = null; diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureException.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureException.java index 6f4795d9dc..d359657252 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureException.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureException.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.procedure2; import java.io.IOException; +import java.util.function.Function; import org.apache.hadoop.ipc.RemoteException; import org.apache.yetus.audience.InterfaceAudience; @@ -37,7 +38,6 @@ import org.apache.hadoop.hbase.util.ForeignExceptionUtil; * their stacks traces and messages overridden to reflect the original 'remote' exception. */ @InterfaceAudience.Private -@InterfaceStability.Evolving @SuppressWarnings("serial") public class RemoteProcedureException extends ProcedureException { @@ -74,6 +74,10 @@ public class RemoteProcedureException extends ProcedureException { return new Exception(cause); } + // NOTE: Does not throw DoNotRetryIOE because it does not + // have access (DNRIOE is in the client module). Use + // MasterProcedureUtil.unwrapRemoteIOException if need to + // throw DNRIOE. public IOException unwrapRemoteIOException() { final Exception cause = unwrapRemoteException(); if (cause instanceof IOException) { diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureBypass.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureBypass.java index d58d57e76e..ee55a1400b 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureBypass.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureBypass.java @@ -179,7 +179,4 @@ public class TestProcedureBypass { } } } - - - } diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto index b2a3bda095..4884adb666 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Master.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto @@ -99,6 +99,7 @@ message MergeTableRegionsResponse { message AssignRegionRequest { required RegionSpecifier region = 1; + optional bool force = 2 [default = false]; } message AssignRegionResponse { @@ -1005,6 +1006,7 @@ message SetTableStateInMetaRequest { // Region at a time. message AssignsRequest { repeated RegionSpecifier region = 1; + optional bool force = 2 [default = false]; } /** Like Admin's AssignRegionResponse except it can @@ -1019,6 +1021,7 @@ message AssignsResponse { */ message UnassignsRequest { repeated RegionSpecifier region = 1; + optional bool force = 2 [default = false]; } /** Like Admin's UnassignRegionResponse except it can @@ -1031,7 +1034,7 @@ message UnassignsResponse { message BypassProcedureRequest { repeated uint64 proc_id = 1; optional uint64 waitTime = 2; // wait time in ms to acquire lock on a procedure - optional bool force = 3; // if true, procedure is marked for bypass even if its executing + optional bool force = 3 [default = false]; // if true, procedure is marked for bypass even if its executing } message BypassProcedureResponse { diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto index 3108f9f57a..13eea8a527 100644 --- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto +++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto @@ -330,6 +330,7 @@ message AssignRegionStateData { optional ServerName target_server = 4; // Current attempt index used for expotential backoff when stuck optional int32 attempt = 5; + optional bool force = 6 [default = false]; } message UnassignRegionStateData { diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java index b34ba8957e..6c8cd9f64a 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java @@ -65,6 +65,7 @@ import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.ServerListener; import org.apache.hadoop.hbase.master.TableStateManager; import org.apache.hadoop.hbase.master.assignment.RegionStates.RegionStateNode; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil; import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.protobuf.ProtobufMagic; @@ -873,7 +874,7 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager { Procedure result = masterServices.getMasterProcedureExecutor().getResult(procId); if (result != null && result.isFailed()) { throw new IOException("Failed to create group table. " + - result.getException().unwrapRemoteIOException()); + MasterProcedureUtil.unwrapRemoteIOException(result)); } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index d7e57e8e2b..8fc57ed63b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -978,7 +978,7 @@ public class HMaster extends HRegionServer implements MasterServices { // as procedures run -- in particular SCPs for crashed servers... One should put up hbase:meta // if it is down. It may take a while to come online. So, wait here until meta if for sure // available. Thats what waitUntilMetaOnline does. - if (!waitUntilMetaOnline()) { + if (!waitForMetaOnline()) { return; } this.assignmentManager.joinCluster(); @@ -1010,7 +1010,7 @@ public class HMaster extends HRegionServer implements MasterServices { // Here we expect hbase:namespace to be online. See inside initClusterSchemaService. // TODO: Fix this. Namespace is a pain being a sort-of system table. Fold it in to hbase:meta. // isNamespace does like isMeta and waits until namespace is onlined before allowing progress. - if (!waitUntilNamespaceOnline()) { + if (!waitForNamespaceOnline()) { return; } status.setStatus("Starting cluster schema service"); @@ -1094,7 +1094,7 @@ public class HMaster extends HRegionServer implements MasterServices { * and we will hold here until operator intervention. */ @VisibleForTesting - public boolean waitUntilMetaOnline() throws InterruptedException { + public boolean waitForMetaOnline() throws InterruptedException { return isRegionOnline(RegionInfoBuilder.FIRST_META_REGIONINFO); } @@ -1135,7 +1135,7 @@ public class HMaster extends HRegionServer implements MasterServices { * @return True if namespace table is up/online. */ @VisibleForTesting - public boolean waitUntilNamespaceOnline() throws InterruptedException { + public boolean waitForNamespaceOnline() throws InterruptedException { List ris = this.assignmentManager.getRegionStates(). getRegionsOfTable(TableName.NAMESPACE_TABLE_NAME); if (ris.isEmpty()) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index bb49066b70..3794652386 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; @@ -1180,7 +1181,8 @@ public class MasterRpcServices extends RSRpcServices if (executor.isFinished(procId)) { builder.setState(GetProcedureResultResponse.State.FINISHED); if (result.isFailed()) { - IOException exception = result.getException().unwrapRemoteIOException(); + IOException exception = + MasterProcedureUtil.unwrapRemoteIOException(result); builder.setException(ForeignExceptionUtil.toProtoForeignException(exception)); } byte[] resultData = result.getResult(); @@ -2334,14 +2336,15 @@ public class MasterRpcServices extends RSRpcServices * Submit the Procedure that gets created by f * @return pid of the submitted Procedure. */ - private long submitProcedure(HBaseProtos.RegionSpecifier rs, Function f) + private long submitProcedure(HBaseProtos.RegionSpecifier rs, boolean force, + BiFunction f) throws UnknownRegionException { RegionInfo ri = getRegionInfo(rs); long pid = Procedure.NO_PROC_ID; if (ri == null) { LOG.warn("No RegionInfo found to match {}", rs); } else { - pid = this.master.getMasterProcedureExecutor().submitProcedure(f.apply(ri)); + pid = this.master.getMasterProcedureExecutor().submitProcedure(f.apply(ri, force)); } return pid; } @@ -2361,9 +2364,10 @@ public class MasterRpcServices extends RSRpcServices MasterProtos.AssignsResponse.Builder responseBuilder = MasterProtos.AssignsResponse.newBuilder(); try { + boolean force = request.getForce(); for (HBaseProtos.RegionSpecifier rs: request.getRegionList()) { - long pid = submitProcedure(rs, - r -> this.master.getAssignmentManager().createAssignProcedure(r)); + long pid = submitProcedure(rs, force, + (r, b) -> this.master.getAssignmentManager().createAssignProcedure(r, b)); responseBuilder.addPid(pid); } return responseBuilder.build(); @@ -2387,9 +2391,10 @@ public class MasterRpcServices extends RSRpcServices MasterProtos.UnassignsResponse.Builder responseBuilder = MasterProtos.UnassignsResponse.newBuilder(); try { + boolean force = request.getForce(); for (HBaseProtos.RegionSpecifier rs: request.getRegionList()) { - long pid = submitProcedure(rs, - ri -> this.master.getAssignmentManager().createUnassignProcedure(ri)); + long pid = submitProcedure(rs, force, + (r, b) -> this.master.getAssignmentManager().createUnassignProcedure(r, b)); responseBuilder.addPid(pid); } return responseBuilder.build(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java index 55aee4ad07..9f310dcd5f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java @@ -99,12 +99,15 @@ public class AssignProcedure extends RegionTransitionProcedure { } public AssignProcedure(final RegionInfo regionInfo) { - super(regionInfo); - this.targetServer = null; + this(regionInfo, null); } public AssignProcedure(final RegionInfo regionInfo, final ServerName destinationServer) { - super(regionInfo); + this(regionInfo, destinationServer, false); + } + + public AssignProcedure(final RegionInfo regionInfo, final ServerName destinationServer, boolean force) { + super(regionInfo, force); this.targetServer = destinationServer; } @@ -138,6 +141,9 @@ public class AssignProcedure extends RegionTransitionProcedure { if (getAttempt() > 0) { state.setAttempt(getAttempt()); } + if (isForce()) { + state.setForce(isForce()); + } serializer.serialize(state.build()); } @@ -148,6 +154,7 @@ public class AssignProcedure extends RegionTransitionProcedure { setTransitionState(state.getTransitionState()); setRegionInfo(ProtobufUtil.toRegionInfo(state.getRegionInfo())); forceNewPlan = state.getForceNewPlan(); + setForce(state.getForce()); if (state.hasTargetServer()) { this.targetServer = ProtobufUtil.toServerName(state.getTargetServer()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index 123abc28e9..3d8f0e7bbc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -677,26 +677,37 @@ public class AssignmentManager implements ServerListener { * Called by things like DisableTableProcedure to get a list of UnassignProcedure * to unassign the regions of the table. */ - public UnassignProcedure[] createUnassignProcedures(final TableName tableName) { - return createUnassignProcedures(regionStates.getTableRegionStateNodes(tableName)); + public AssignProcedure createAssignProcedure(final RegionInfo regionInfo) { + return createAssignProcedure(regionInfo, null, false); } - public AssignProcedure createAssignProcedure(final RegionInfo regionInfo) { - AssignProcedure proc = new AssignProcedure(regionInfo); - proc.setOwner(getProcedureEnvironment().getRequestUser().getShortName()); - return proc; + public AssignProcedure createAssignProcedure(final RegionInfo regionInfo, boolean force) { + return createAssignProcedure(regionInfo, null, force); + } + + public AssignProcedure createAssignProcedure(final RegionInfo regionInfo, + ServerName targetServer) { + return createAssignProcedure(regionInfo, targetServer, false); } public AssignProcedure createAssignProcedure(final RegionInfo regionInfo, - final ServerName targetServer) { - AssignProcedure proc = new AssignProcedure(regionInfo, targetServer); + final ServerName targetServer, boolean force) { + AssignProcedure proc = new AssignProcedure(regionInfo, targetServer, force); proc.setOwner(getProcedureEnvironment().getRequestUser().getShortName()); return proc; } public UnassignProcedure createUnassignProcedure(final RegionInfo regionInfo) { return createUnassignProcedure(regionInfo, null, false); + } + public UnassignProcedure createUnassignProcedure(final RegionInfo regionInfo, + boolean force) { + return createUnassignProcedure(regionInfo, null, force); + } + + public UnassignProcedure[] createUnassignProcedures(final TableName tableName) { + return createUnassignProcedures(regionStates.getTableRegionStateNodes(tableName)); } UnassignProcedure createUnassignProcedure(final RegionInfo regionInfo, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java index 3b0735e46e..6a71c17408 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java @@ -111,6 +111,12 @@ public abstract class RegionTransitionProcedure */ private RegionInfo regionInfo; + /** + * this data member must also be persisted. + * @see #regionInfo + */ + private boolean force; + /** * Like {@link #regionInfo}, the expectation is that subclasses persist the value of this * data member. It is used doing backoff when Procedure gets stuck. @@ -120,8 +126,9 @@ public abstract class RegionTransitionProcedure // Required by the Procedure framework to create the procedure on replay public RegionTransitionProcedure() {} - public RegionTransitionProcedure(final RegionInfo regionInfo) { + public RegionTransitionProcedure(final RegionInfo regionInfo, boolean force) { this.regionInfo = regionInfo; + this.force = force; } @VisibleForTesting @@ -134,7 +141,7 @@ public abstract class RegionTransitionProcedure * {@link #deserializeStateData(ProcedureStateSerializer)} method. Expectation is that * subclasses will persist `regioninfo` in their * {@link #serializeStateData(ProcedureStateSerializer)} method and then restore `regionInfo` on - * deserialization by calling. + * deserialization by calling this. */ protected void setRegionInfo(final RegionInfo regionInfo) { this.regionInfo = regionInfo; @@ -142,9 +149,21 @@ public abstract class RegionTransitionProcedure /** * This setter is for subclasses to call in their - * {@link #deserializeStateData(ProcedureStateSerializer)} method. - * @see #setRegionInfo(RegionInfo) + * {@link #deserializeStateData(ProcedureStateSerializer)} method. Expectation is that + * subclasses will persist `force` in their + * {@link #serializeStateData(ProcedureStateSerializer)} method and then restore `force` on + * deserialization by calling this. */ + protected void setForce(boolean force) { + this.force = force; + } + + + /** + * This setter is for subclasses to call in their + * {@link #deserializeStateData(ProcedureStateSerializer)} method. + * @see #setRegionInfo(RegionInfo) + */ protected void setAttempt(int attempt) { this.attempt = attempt; } @@ -170,6 +189,11 @@ public abstract class RegionTransitionProcedure sb.append(getTableName()); sb.append(", region="); sb.append(getRegionInfo() == null? null: getRegionInfo().getEncodedName()); + if (isForce()) { + // Only log if set. + sb.append(", force="); + sb.append(isForce()); + } } public RegionStateNode getRegionState(final MasterProcedureEnv env) { @@ -308,12 +332,17 @@ public abstract class RegionTransitionProcedure final AssignmentManager am = env.getAssignmentManager(); final RegionStateNode regionNode = getRegionState(env); if (!am.addRegionInTransition(regionNode, this)) { - String msg = String.format( - "There is already another procedure running on this region this=%s owner=%s", - this, regionNode.getProcedure()); - LOG.warn(msg + " " + this + "; " + regionNode.toShortString()); - setAbortFailure(getClass().getSimpleName(), msg); - return null; + if (this.isForce()) { + regionNode.getProcedure().getProcId(), getProcId()); + regionNode.unsetProcedure(regionNode.getProcedure()); + } else { + String msg = String.format("%s owned by pid=%d, CANNOT run 'this' (pid=%d).", + regionNode.getRegionInfo().getEncodedName(), + regionNode.getProcedure().getProcId(), getProcId()); + LOG.warn(msg); + setAbortFailure(getClass().getSimpleName(), msg); + return null; + } } try { boolean retry; @@ -469,4 +498,23 @@ public abstract class RegionTransitionProcedure // should not be called for region operation until we modified the open/close region procedure throw new UnsupportedOperationException(); } + + @Override + protected void bypass(MasterProcedureEnv env) { + // This override is just so I can write a note on how bypass is done in + // RTP. For RTP procedures -- i.e. assign/unassign -- if bypass is called, + // we intentionally do NOT cleanup our state. We leave a reference to the + // bypassed Procedure in the RegionStateNode. Doing this makes it so the + // RSN is in an odd state. The bypassed Procedure is finished but no one + // else can make progress on this RSN entity (see the #execute above where + // we check the RSN to see if an already registered procedure and if so, + // we exit without proceeding). This is done to intentionally block + // subsequent Procedures from running. Only a Procedure with the 'force' flag + // set can overwrite the RSN and make progress. + super.bypass(env); + } + + boolean isForce() { + return this.force; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/UnassignProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/UnassignProcedure.java index 4f58a0f305..c354e2019d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/UnassignProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/UnassignProcedure.java @@ -84,10 +84,6 @@ public class UnassignProcedure extends RegionTransitionProcedure { */ protected volatile ServerName destinationServer; - // TODO: should this be in a reassign procedure? - // ...and keep unassign for 'disable' case? - private boolean force; - /** * Whether deleting the region from in-memory states after unassigning the region. */ @@ -111,10 +107,9 @@ public class UnassignProcedure extends RegionTransitionProcedure { public UnassignProcedure(final RegionInfo regionInfo, final ServerName hostingServer, final ServerName destinationServer, final boolean force, final boolean removeAfterUnassigning) { - super(regionInfo); + super(regionInfo, force); this.hostingServer = hostingServer; this.destinationServer = destinationServer; - this.force = force; this.removeAfterUnassigning = removeAfterUnassigning; // we don't need REGION_TRANSITION_QUEUE, we jump directly to sending the request @@ -147,7 +142,7 @@ public class UnassignProcedure extends RegionTransitionProcedure { if (this.destinationServer != null) { state.setDestinationServer(ProtobufUtil.toServerName(destinationServer)); } - if (force) { + if (isForce()) { state.setForce(true); } if (removeAfterUnassigning) { @@ -167,7 +162,7 @@ public class UnassignProcedure extends RegionTransitionProcedure { setTransitionState(state.getTransitionState()); setRegionInfo(ProtobufUtil.toRegionInfo(state.getRegionInfo())); this.hostingServer = ProtobufUtil.toServerName(state.getHostingServer()); - force = state.getForce(); + setForce(state.getForce()); if (state.hasDestinationServer()) { this.destinationServer = ProtobufUtil.toServerName(state.getDestinationServer()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureUtil.java index 58263d3044..1e488b6185 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureUtil.java @@ -19,9 +19,12 @@ package org.apache.hadoop.hbase.master.procedure; import java.io.IOException; import java.util.regex.Pattern; + +import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureException; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.NonceKey; @@ -176,4 +179,17 @@ public final class MasterProcedureUtil { public static int getServerPriority(ServerProcedureInterface proc) { return proc.hasMetaTableRegion() ? 100 : 1; } + + /** + * This is a version of unwrapRemoteIOException that can do DoNotRetryIOE. + * We need to throw DNRIOE to clients if a failed Procedure else they will + * keep trying. The default proc.getException().unwrapRemoteException + * doesn't have access to DNRIOE from the procedure2 module. + */ + public static IOException unwrapRemoteIOException(Procedure proc) { + Exception e = proc.getException().unwrapRemoteException(); + // Do not retry ProcedureExceptions! + return (e instanceof ProcedureException)? new DoNotRetryIOException(e): + proc.getException().unwrapRemoteIOException(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureDescriber.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureDescriber.java index 41b30ad5a2..f9fdad8aba 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureDescriber.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureDescriber.java @@ -69,7 +69,8 @@ public class ProcedureDescriber { description.put("LAST_UPDATE", new Date(proc.getLastUpdate())); if (proc.isFailed()) { - description.put("ERRORS", proc.getException().unwrapRemoteIOException().getMessage()); + description.put("ERRORS", + MasterProcedureUtil.unwrapRemoteIOException(proc).getMessage()); } description.put("PARAMETERS", parametersToObject(proc)); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java index 2011c0b9de..f771210b87 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java @@ -99,7 +99,7 @@ public abstract class ProcedurePrepareLatch { @Override protected void countDown(final Procedure proc) { if (proc.hasException()) { - exception = proc.getException().unwrapRemoteIOException(); + exception = MasterProcedureUtil.unwrapRemoteIOException(proc); } latch.countDown(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java index f72905bb52..bd1fe66821 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java @@ -27,14 +27,19 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.exceptions.TimeoutIOException; import org.apache.hadoop.hbase.master.assignment.RegionStates; import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureAbortedException; +import org.apache.hadoop.hbase.procedure2.ProcedureException; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.RemoteProcedureException; import org.apache.hadoop.hbase.quotas.MasterQuotaManager; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.ipc.RemoteException; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; import org.slf4j.Logger; @@ -160,9 +165,10 @@ public final class ProcedureSyncWait { throw new IOException("The Master is Aborting"); } + // If the procedure fails, we should always have an exception captured. Throw it. + // Needs to be an IOE to get out of here. if (proc.hasException()) { - // If the procedure fails, we should always have an exception captured. Throw it. - throw proc.getException().unwrapRemoteIOException(); + throw MasterProcedureUtil.unwrapRemoteIOException(proc); } else { return proc.getResult(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java index 2916cc410c..0ccf26e9c2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java @@ -108,7 +108,7 @@ public class TestMetaTableAccessor { @Test public void testIsMetaWhenAllHealthy() throws InterruptedException { HMaster m = UTIL.getMiniHBaseCluster().getMaster(); - assertTrue(m.waitUntilMetaOnline()); + assertTrue(m.waitForMetaOnline()); } @Test @@ -117,7 +117,7 @@ public class TestMetaTableAccessor { int index = UTIL.getMiniHBaseCluster().getServerWithMeta(); HRegionServer rsWithMeta = UTIL.getMiniHBaseCluster().getRegionServer(index); rsWithMeta.abort("TESTING"); - assertTrue(m.waitUntilMetaOnline()); + assertTrue(m.waitForMetaOnline()); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionBypass.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionBypass.java new file mode 100644 index 0000000000..300e9b2b83 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionBypass.java @@ -0,0 +1,179 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.assignment; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import static org.junit.Assert.assertTrue; + +/** + * Tests bypass on a region assign/unassign + */ +@Category({LargeTests.class}) +public class TestRegionBypass { + private final static Logger LOG = LoggerFactory.getLogger(TestRegionBypass.class); + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestRegionBypass.class); + + @Rule + public TestName name = new TestName(); + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private TableName tableName; + + @BeforeClass + public static void startCluster() throws Exception { + TEST_UTIL.startMiniCluster(2); + } + + @AfterClass + public static void stopCluster() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void before() throws IOException { + this.tableName = TableName.valueOf(this.name.getMethodName()); + // Create a table. Has one region at least. + TEST_UTIL.createTable(this.tableName, Bytes.toBytes("cf")); + + } + + @Test + public void testBypass() throws IOException { + Admin admin = TEST_UTIL.getAdmin(); + List regions = admin.getRegions(this.tableName); + for (RegionInfo ri: regions) { + admin.unassign(ri.getRegionName(), false); + } + List pids = new ArrayList<>(regions.size()); + for (RegionInfo ri: regions) { + Procedure p = new StallingAssignProcedure(ri); + pids.add(TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(). + submitProcedure(p)); + } + for (Long pid: pids) { + while (!TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor().isStarted(pid)) { + Thread.currentThread().yield(); + } + } + // Call bypass on all. We should be stuck in the dispatch at this stage. + List> ps = + TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor().getProcedures(); + for (Procedure p: ps) { + if (p instanceof StallingAssignProcedure) { + List bs = TEST_UTIL.getHbck(). + bypassProcedure(Arrays.asList(p.getProcId()), 0, false); + for (Boolean b: bs) { + LOG.info("BYPASSED {} {}", p.getProcId(), b); + } + } + } + // Countdown the latch so its not hanging out. + for (Procedure p: ps) { + if (p instanceof StallingAssignProcedure) { + ((StallingAssignProcedure)p).latch.countDown(); + } + } + // Try and assign WITHOUT force flag. Should fail!. + for (RegionInfo ri: regions) { + try { + admin.assign(ri.getRegionName()); + } catch (Throwable dnrioe) { + // Expected + LOG.info("Expected {}", dnrioe); + } + } + while (!TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(). + getActiveProcIds().isEmpty()) { + Thread.currentThread().yield(); + } + // Now assign with the force flag. + for (RegionInfo ri: regions) { + TEST_UTIL.getHbck().assigns(Arrays.asList(ri.getEncodedName()), true); + } + while (!TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(). + getActiveProcIds().isEmpty()) { + Thread.currentThread().yield(); + } + for (RegionInfo ri: regions) { + assertTrue(ri.toString(), TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager(). + getRegionStates().isRegionOnline(ri)); + } + } + + /** + * An AssignProcedure that Stalls just before the finish. + */ + public static class StallingAssignProcedure extends AssignProcedure { + public final CountDownLatch latch = new CountDownLatch(2); + + public StallingAssignProcedure() { + super(); + } + + public StallingAssignProcedure(RegionInfo regionInfo) { + super(regionInfo); + } + + @Override + void setTransitionState(MasterProcedureProtos.RegionTransitionState state) { + if (state == MasterProcedureProtos.RegionTransitionState.REGION_TRANSITION_DISPATCH) { + try { + LOG.info("LATCH2 {}", this.latch.getCount()); + this.latch.await(); + LOG.info("LATCH3 {}", this.latch.getCount()); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } else if (state == MasterProcedureProtos.RegionTransitionState.REGION_TRANSITION_QUEUE) { + // Set latch. + LOG.info("LATCH1 {}", this.latch.getCount()); + this.latch.countDown(); + } + super.setTransitionState(state); + } + } +} diff --git a/hbase-shell/src/main/ruby/shell/commands/list_procedures.rb b/hbase-shell/src/main/ruby/shell/commands/list_procedures.rb index 68842a0410..ac39a26fd7 100644 --- a/hbase-shell/src/main/ruby/shell/commands/list_procedures.rb +++ b/hbase-shell/src/main/ruby/shell/commands/list_procedures.rb @@ -31,12 +31,11 @@ EOF end def command - formatter.header(%w[Id Name State Submitted_Time Last_Update Parameters]) - + formatter.header(%w[PID Name State Submitted_Time Last_Update Parameters]) list = JSON.parse(admin.list_procedures) list.each do |proc| - submitted_time = Time.at(Integer(proc['submittedTime']) / 1000).to_s - last_update = Time.at(Integer(proc['lastUpdate']) / 1000).to_s + submitted_time = Time.iso8601(Integer(proc['submittedTime']) / 1000).to_s + last_update = Time.iso8601(Integer(proc['lastUpdate']) / 1000).to_s formatter.row([proc['procId'], proc['className'], proc['state'], submitted_time, last_update, proc['stateMessage']]) end -- 2.16.3