From c94ba6d20481890dc766d6aa245420fb1d17ca73 Mon Sep 17 00:00:00 2001 From: Ben Lau Date: Tue, 20 Feb 2018 14:08:02 -0800 Subject: [PATCH] HBASE-15911 NPE in AssignmentManager.onRegionTransition after Master restart --- .../org/apache/hadoop/hbase/master/HMaster.java | 12 +- .../hadoop/hbase/master/MasterRpcServices.java | 35 ++--- .../assignment/MergeTableRegionsProcedure.java | 3 +- .../assignment/SplitTableRegionProcedure.java | 3 +- .../master/procedure/CreateNamespaceProcedure.java | 2 +- .../master/procedure/DeleteNamespaceProcedure.java | 2 +- .../hbase/master/procedure/ProcedureSyncWait.java | 12 +- .../hadoop/hbase/quotas/MasterQuotaManager.java | 17 +- .../TestSplitTransactionOnCluster.java | 174 +++++++++++++++++++-- 9 files changed, 211 insertions(+), 49 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 02fbc028bb..a1c2418f7c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -849,6 +849,9 @@ public class HMaster extends HRegionServer implements MasterServices { status.setStatus("Initializing master coprocessors"); this.cpHost = new MasterCoprocessorHost(this, this.conf); + this.quotaManager = new MasterQuotaManager(this); + this.assignmentManager.setRegionStateListener(quotaManager); + // start up all service threads. status.setStatus("Initializing master service threads"); startServiceThreads(); @@ -939,7 +942,7 @@ public class HMaster extends HRegionServer implements MasterServices { status.setStatus("Assign meta replicas"); metaBootstrap.assignMetaReplicas(); status.setStatus("Starting quota manager"); - initQuotaManager(); + quotaManager.start(); if (QuotaUtil.isQuotaEnabled(conf)) { // Create the quota snapshot notifier spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier(); @@ -1060,13 +1063,6 @@ public class HMaster extends HRegionServer implements MasterServices { } } - void initQuotaManager() throws IOException { - MasterQuotaManager quotaManager = new MasterQuotaManager(this); - this.assignmentManager.setRegionStateListener(quotaManager); - quotaManager.start(); - this.quotaManager = quotaManager; - } - SpaceQuotaSnapshotNotifier createQuotaSnapshotNotifier() { SpaceQuotaSnapshotNotifier notifier = SpaceQuotaSnapshotNotifierFactory.getInstance().create(getConfiguration()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 377a9c6dec..01770de790 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -2157,28 +2157,25 @@ public class MasterRpcServices extends RSRpcServices RpcController controller, GetSpaceQuotaRegionSizesRequest request) throws ServiceException { try { master.checkInitialized(); - MasterQuotaManager quotaManager = this.master.getMasterQuotaManager(); + MasterQuotaManager quotaManager = master.getMasterQuotaManager(); GetSpaceQuotaRegionSizesResponse.Builder builder = GetSpaceQuotaRegionSizesResponse.newBuilder(); - if (quotaManager != null) { - Map regionSizes = quotaManager.snapshotRegionSizes(); - Map regionSizesByTable = new HashMap<>(); - // Translate hregioninfo+long -> tablename+long - for (Entry entry : regionSizes.entrySet()) { - final TableName tableName = entry.getKey().getTable(); - Long prevSize = regionSizesByTable.get(tableName); - if (prevSize == null) { - prevSize = 0L; - } - regionSizesByTable.put(tableName, prevSize + entry.getValue()); - } - // Serialize them into the protobuf - for (Entry tableSize : regionSizesByTable.entrySet()) { - builder.addSizes(RegionSizes.newBuilder() - .setTableName(ProtobufUtil.toProtoTableName(tableSize.getKey())) - .setSize(tableSize.getValue()).build()); + Map regionSizes = quotaManager.snapshotRegionSizes(); + Map regionSizesByTable = new HashMap<>(); + // Translate hregioninfo+long -> tablename+long + for (Entry entry : regionSizes.entrySet()) { + final TableName tableName = entry.getKey().getTable(); + Long prevSize = regionSizesByTable.get(tableName); + if (prevSize == null) { + prevSize = 0L; } - return builder.build(); + regionSizesByTable.put(tableName, prevSize + entry.getValue()); + } + // Serialize them into the protobuf + for (Entry tableSize : regionSizesByTable.entrySet()) { + builder.addSizes(RegionSizes.newBuilder() + .setTableName(ProtobufUtil.toProtoTableName(tableSize.getKey())) + .setSize(tableSize.getValue()).build()); } return builder.build(); } catch (Exception e) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index 4bccab71f3..41c864ea29 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan; import org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil; +import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait; import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; @@ -533,7 +534,7 @@ public class MergeTableRegionsProcedure } // TODO: Clean up split and merge. Currently all over the place. try { - env.getMasterServices().getMasterQuotaManager().onRegionMerged(this.mergedRegion); + ProcedureSyncWait.getMasterQuotaManager(env).onRegionMerged(this.mergedRegion); } catch (QuotaExceededException e) { env.getAssignmentManager().getRegionNormalizer().planSkipped(this.mergedRegion, NormalizationPlan.PlanType.MERGE); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java index be0741d873..ee5fe6fbde 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java @@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan; import org.apache.hadoop.hbase.master.procedure.AbstractStateMachineRegionProcedure; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil; +import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait; import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; import org.apache.hadoop.hbase.quotas.QuotaExceededException; @@ -486,7 +487,7 @@ public class SplitTableRegionProcedure // TODO: Clean up split and merge. Currently all over the place. // Notify QuotaManager and RegionNormalizer try { - env.getMasterServices().getMasterQuotaManager().onRegionSplit(this.getParentRegion()); + ProcedureSyncWait.getMasterQuotaManager(env).onRegionSplit(this.getParentRegion()); } catch (QuotaExceededException e) { env.getAssignmentManager().getRegionNormalizer().planSkipped(this.getParentRegion(), NormalizationPlan.PlanType.SPLIT); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java index 5eb56f5456..a95e2140d2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java @@ -245,7 +245,7 @@ public class CreateNamespaceProcedure final MasterProcedureEnv env, final NamespaceDescriptor nsDescriptor) throws IOException { if (env.getMasterServices().isInitialized()) { - env.getMasterServices().getMasterQuotaManager().setNamespaceQuota(nsDescriptor); + ProcedureSyncWait.getMasterQuotaManager(env).setNamespaceQuota(nsDescriptor); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java index 1c587eb55d..75be019411 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java @@ -318,7 +318,7 @@ public class DeleteNamespaceProcedure protected static void removeNamespaceQuota( final MasterProcedureEnv env, final String namespaceName) throws IOException { - env.getMasterServices().getMasterQuotaManager().removeNamespaceQuota(namespaceName); + ProcedureSyncWait.getMasterQuotaManager(env).removeNamespaceQuota(namespaceName); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java index df0875ec1e..494e584a06 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.master.assignment.RegionStates; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.quotas.MasterQuotaManager; +import org.apache.hadoop.hbase.quotas.QuotaUtil; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -219,13 +220,20 @@ public final class ProcedureSyncWait { } } - protected static MasterQuotaManager getMasterQuotaManager(final MasterProcedureEnv env) + public static MasterQuotaManager getMasterQuotaManager(final MasterProcedureEnv env) throws IOException { return ProcedureSyncWait.waitFor(env, "quota manager to be available", new ProcedureSyncWait.Predicate() { @Override public MasterQuotaManager evaluate() throws IOException { - return env.getMasterServices().getMasterQuotaManager(); + MasterQuotaManager masterQuotaManager = env.getMasterServices().getMasterQuotaManager(); + if (masterQuotaManager != null) { + if (!QuotaUtil.isQuotaEnabled(env.getMasterServices().getConfiguration()) || + masterQuotaManager.isQuotaInitialized()) { + return masterQuotaManager; + } + } + return null; } }); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java index 6783e7da0b..69f8ecf914 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java @@ -31,6 +31,7 @@ import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.PleaseHoldException; import org.apache.hadoop.hbase.RegionStateListener; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; @@ -66,7 +67,7 @@ public class MasterQuotaManager implements RegionStateListener { private NamedLock namespaceLocks; private NamedLock tableLocks; private NamedLock userLocks; - private boolean initialized = false; + private volatile boolean initialized = false; private NamespaceAuditor namespaceQuotaManager; private ConcurrentHashMap regionSizes; @@ -369,6 +370,8 @@ public class MasterQuotaManager implements RegionStateListener { public void onRegionMerged(RegionInfo mergedRegion) throws IOException { if (initialized) { namespaceQuotaManager.updateQuotaForRegionMerge(mergedRegion); + } else if (QuotaUtil.isQuotaEnabled(masterServices.getConfiguration())){ + throw new PleaseHoldException("Master quota manager not ready to process region merges yet"); } } @@ -376,6 +379,8 @@ public class MasterQuotaManager implements RegionStateListener { public void onRegionSplit(RegionInfo hri) throws IOException { if (initialized) { namespaceQuotaManager.checkQuotaToSplitRegion(hri); + } else if (QuotaUtil.isQuotaEnabled(masterServices.getConfiguration())){ + throw new PleaseHoldException("Master quota manager not ready to process region splits yet"); } } @@ -478,6 +483,8 @@ public class MasterQuotaManager implements RegionStateListener { public void onRegionSplitReverted(RegionInfo hri) throws IOException { if (initialized) { this.namespaceQuotaManager.removeRegionFromNamespaceUsage(hri); + } else if (QuotaUtil.isQuotaEnabled(masterServices.getConfiguration())){ + throw new PleaseHoldException("Master quota manager not ready to process region splits yet"); } } @@ -527,19 +534,21 @@ public class MasterQuotaManager implements RegionStateListener { @VisibleForTesting void initializeRegionSizes() { + assert initialized == false; assert regionSizes == null; + this.initialized = true; this.regionSizes = new ConcurrentHashMap<>(); } public void addRegionSize(RegionInfo hri, long size, long time) { - if (regionSizes == null) { + if (!initialized) { return; } regionSizes.put(hri, new SizeSnapshotWithTimestamp(size, time)); } public Map snapshotRegionSizes() { - if (regionSizes == null) { + if (!initialized) { return EMPTY_MAP; } @@ -551,7 +560,7 @@ public class MasterQuotaManager implements RegionStateListener { } int pruneEntriesOlderThan(long timeToPruneBefore) { - if (regionSizes == null) { + if (!initialized) { return 0; } int numEntriesRemoved = 0; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java index b2a88b0bde..313478e916 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java @@ -32,6 +32,8 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -53,14 +55,17 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Consistency; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TestReplicasClient.SlowMeCopro; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor; import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.MasterObserver; @@ -75,6 +80,7 @@ import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.master.assignment.AssignmentManager; import org.apache.hadoop.hbase.master.assignment.RegionStates; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.quotas.QuotaUtil; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -132,6 +138,8 @@ public class TestSplitTransactionOnCluster { @BeforeClass public static void before() throws Exception { TESTING_UTIL.getConfiguration().setInt(HConstants.HBASE_BALANCER_PERIOD, 60000); + TESTING_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, + DelayedInitializationObserver.class.getName()); TESTING_UTIL.startMiniCluster(1, NB_SERVERS, null, MyMaster.class, null); } @@ -493,6 +501,132 @@ public class TestSplitTransactionOnCluster { } } + /** + * Verifies HBASE-15911. When master is starting, region splits should not cause the master to + * fail eg NPE due to uninitialized state in the AssignmentManager or elsewhere. + * @throws Exception + */ + @Test(timeout = 120000) + public void testSplitBeforeMasterInitialization() throws Exception { + final TableName tableName = TableName.valueOf("testSplitBeforeMasterInitialization"); + Table t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY); + List regions = cluster.getRegions(tableName); + RegionInfo regionToSplit1 = getAndCheckSingleTableRegion(regions); + + LOG.info("Waiting for master to be fully shutdown"); + ServerName masterName = TESTING_UTIL.getMiniHBaseCluster().getMaster().getServerName(); + TESTING_UTIL.getMiniHBaseCluster().stopMaster(masterName); + TESTING_UTIL.getMiniHBaseCluster().waitForMasterToStop(masterName, 15000); + + try { + LOG.info("Starting master with delayed initialization and quota support"); + DelayedInitializationObserver.enable(); + TESTING_UTIL.getMiniHBaseCluster().getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + TESTING_UTIL.getMiniHBaseCluster().startMaster(); + + RegionLocator regionLocator = ((HTable) t).getRegionLocator(); + + int uncaughtTransitionExceptionsBefore = getUncaughtTransitionExceptions(); + + // Split a region while master is initializing. + LOG.info("Splitting region: " + regionToSplit1.getEncodedName() + " ON " + + regionLocator.getRegionLocation(regionToSplit1.getStartKey())); + TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false); + admin.splitRegion(regionToSplit1.getEncodedNameAsBytes()); + + // Long enough for the split to be reported back to an uninitialized master. + Thread.sleep(5000); + + LOG.info("Resuming master initialization"); + DelayedInitializationObserver.startSignal.countDown(); + int uncaughtTransitionExceptionsAfter = getUncaughtTransitionExceptions(); + + // Wait some time and ensure that master didn't die and there are now 2 regions + Thread.sleep(10000); + LOG.info("Performing checks now"); + assertEquals(2, regionLocator.getAllRegionLocations().size()); + assertTrue(TESTING_UTIL.getMiniHBaseCluster().getMaster().isInitialized()); + assertEquals(1, TESTING_UTIL.getMiniHBaseCluster().getLiveMasterThreads().size()); + assertEquals(uncaughtTransitionExceptionsBefore, uncaughtTransitionExceptionsAfter); + } finally { + DelayedInitializationObserver.reset(); + TESTING_UTIL.getMiniHBaseCluster().getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, false); + } + + LOG.info("Waiting for master to be fully shutdown"); + masterName = TESTING_UTIL.getMiniHBaseCluster().getMaster().getServerName(); + TESTING_UTIL.getMiniHBaseCluster().stopMaster(masterName); + TESTING_UTIL.getMiniHBaseCluster().waitForMasterToStop(masterName, 15000); + + /* Now test without quotas, region split should succeed + */ + + try { + LOG.info("Starting master with delayed initialization"); + DelayedInitializationObserver.enable(); + TESTING_UTIL.getMiniHBaseCluster().startMaster(); + + RegionLocator regionLocator = ((HTable)t).getRegionLocator(); + RegionInfo regionToSplit2 = regionLocator.getRegionLocation(regionToSplit1.getStartKey(), true) + .getRegion(); + + int uncaughtTransitionExceptionsBefore = getUncaughtTransitionExceptions(); + + // Split a region, should succeed + LOG.info("Splitting region: " + regionToSplit2.getEncodedName() + " ON " + + regionLocator.getRegionLocation(regionToSplit2.getStartKey())); + TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false); + admin.splitRegion(regionToSplit2.getEncodedNameAsBytes()); + + // Long enough for the split to be reported back to an uninitialized master. + Thread.sleep(5000); + + LOG.info("Resuming master initialization"); + DelayedInitializationObserver.startSignal.countDown(); + + int uncaughtTransitionExceptionsAfter = getUncaughtTransitionExceptions(); + + // Wait some time and ensure that master didn't die and there are now 3 regions + Thread.sleep(10000); + LOG.info("Performing checks now"); + assertEquals(3, regionLocator.getAllRegionLocations().size()); + assertTrue(TESTING_UTIL.getMiniHBaseCluster().getMaster().isInitialized()); + assertEquals(1, TESTING_UTIL.getMiniHBaseCluster().getLiveMasterThreads().size()); + assertEquals(uncaughtTransitionExceptionsBefore, uncaughtTransitionExceptionsAfter); + } finally { + DelayedInitializationObserver.reset(); + } + } + + public static class DelayedInitializationObserver implements MasterCoprocessor, MasterObserver { + private static volatile boolean enabled = false; + private static volatile CountDownLatch startSignal = new CountDownLatch(1); + + @Override + public void preMasterInitialization(ObserverContext ctx) + throws IOException { + if (!enabled) { + return; + } + LOG.info("Delaying master initialization in coproc"); + try { + startSignal.await(); + } catch (InterruptedException e) { + LOG.error("Unexpected interrupt", e); + } + LOG.info("Resuming master initialization in coproc"); + } + + public static void enable() { + enabled = true; + } + + public static void reset() { + enabled = false; + startSignal = new CountDownLatch(1); + } + } + /** * Verifies HBASE-5806. Here the case is that splitting is completed but before the * CJ could remove the parent region the master is killed and restarted. @@ -948,6 +1082,14 @@ public class TestSplitTransactionOnCluster { return t; } + private int getUncaughtTransitionExceptions() { + MyMaster master = (MyMaster) TESTING_UTIL.getMiniHBaseCluster().getMaster(); + if (master == null) return 0; + MyMasterRpcServices rpcServices = ((MyMasterRpcServices) master.getMasterRpcServices()); + if (rpcServices == null) return 0; + return rpcServices.uncaughtTransitionExceptions.get(); + } + // Make it public so that JVMClusterUtil can access it. public static class MyMaster extends HMaster { public MyMaster(Configuration conf) throws IOException, KeeperException, InterruptedException { @@ -962,6 +1104,7 @@ public class TestSplitTransactionOnCluster { static class MyMasterRpcServices extends MasterRpcServices { static AtomicBoolean enabled = new AtomicBoolean(false); + final AtomicInteger uncaughtTransitionExceptions = new AtomicInteger(); private HMaster myMaster; public MyMasterRpcServices(HMaster master) throws IOException { @@ -972,21 +1115,28 @@ public class TestSplitTransactionOnCluster { @Override public ReportRegionStateTransitionResponse reportRegionStateTransition(RpcController c, ReportRegionStateTransitionRequest req) throws ServiceException { - ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(c, req); - if (enabled.get() && req.getTransition(0).getTransitionCode().equals( - TransitionCode.READY_TO_SPLIT) && !resp.hasErrorMessage()) { - RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates(); - for (RegionStates.RegionStateNode regionState: - regionStates.getRegionsInTransition()) { - /* TODO!!!! - // Find the merging_new region and remove it - if (regionState.isSplittingNew()) { - regionStates.deleteRegion(regionState.getRegion()); + try { + ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(c, req); + if (enabled.get() && req.getTransition(0).getTransitionCode().equals( + TransitionCode.READY_TO_SPLIT) && !resp.hasErrorMessage()) { + RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates(); + for (RegionStates.RegionStateNode regionState: + regionStates.getRegionsInTransition()) { + /* TODO!!!! + // Find the merging_new region and remove it + if (regionState.isSplittingNew()) { + regionStates.deleteRegion(regionState.getRegion()); + } + */ } - */ } + return resp; + } catch (ServiceException e1) { + throw e1; + } catch (Exception e2) { + uncaughtTransitionExceptions.addAndGet(1); + throw e2; } - return resp; } } -- 2.14.3 (Apple Git-98)