From 8ab40e183ef6f5957d4635f44387a62a6450cdba Mon Sep 17 00:00:00 2001 From: Thiruvel Thirumoolan Date: Thu, 22 Feb 2018 12:51:08 -0800 Subject: [PATCH] HBASE-20001 cleanIfNoMetaEntry() uses encoded instead of region name to lookup region --- .../hadoop/hbase/master/AssignmentManager.java | 14 +- .../apache/hadoop/hbase/master/RegionStates.java | 45 +++-- .../TestRegionMergeTransactionOnCluster.java | 183 ++++++++++++++++++++- .../TestSplitTransactionOnCluster.java | 124 ++++++++------ 4 files changed, 298 insertions(+), 68 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index b8c088a583..92ad2f2bb2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -255,10 +255,12 @@ public class AssignmentManager extends ZooKeeperListener { private final RegionStateStore regionStateStore; /** - * For testing only! Set to true to skip handling of split. + * For testing only! Set to true to skip handling of split and merge. */ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="MS_SHOULD_BE_FINAL") public static boolean TEST_SKIP_SPLIT_HANDLING = false; + @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="MS_SHOULD_BE_FINAL") + public static boolean TEST_SKIP_MERGE_HANDLING = false; /** Listeners that are called on assignment events. */ private List listeners = new CopyOnWriteArrayList(); @@ -4054,6 +4056,11 @@ public class AssignmentManager extends ZooKeeperListener { mergingRegions.put(encodedName, new PairOfSameType(a, b)); } else if (code == TransitionCode.MERGED) { + + if (TEST_SKIP_MERGE_HANDLING) { + return "Skipping merge message, TEST_SKIP_MERGE_HANDLING is set for merge parent: " + p; + } + mergingRegions.remove(encodedName); regionOffline(a, State.MERGED); regionOffline(b, State.MERGED); @@ -4180,6 +4187,11 @@ public class AssignmentManager extends ZooKeeperListener { regionStates.updateRegionState(hri_b, State.MERGING); regionStates.updateRegionState(p, State.MERGING_NEW, sn); + if (TEST_SKIP_MERGE_HANDLING) { + LOG.warn("Skipping merge message, TEST_SKIP_MERGE_HANDLING is set for merge parent: " + p); + return true; // return true so that the merging node stays + } + if (et != EventType.RS_ZK_REGION_MERGED) { this.mergingRegions.put(encodedName, new PairOfSameType(hri_a, hri_b)); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java index 7ce62573d6..009ab05e69 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ConfigUtil; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.zookeeper.ZKAssign; @@ -174,12 +175,17 @@ public class RegionStates { static final String LOG_SPLIT_TIME = "hbase.master.maximum.logsplit.keeptime"; static final long DEFAULT_LOG_SPLIT_TIME = 7200000L; // 2 hours + boolean useZK = true; // Is it ZK based assignment? + RegionStates(final MasterServices master, final TableStateManager tableStateManager, final ServerManager serverManager, final RegionStateStore regionStateStore) { this.tableStateManager = tableStateManager; this.regionStateStore = regionStateStore; this.serverManager = serverManager; this.server = master; + if(server != null) { + useZK = ConfigUtil.useZKForAssignment(server.getConfiguration()); + } } /** @@ -726,7 +732,7 @@ public class RegionStates { public List serverOffline(final ZooKeeperWatcher watcher, final ServerName sn) { // Offline all regions on this server not already in transition. List rits = new ArrayList(); - Set regionsToCleanIfNoMetaEntry = new HashSet(); + Set regionsToClean = new HashSet(); // Offline regions outside the loop and synchronized block to avoid // ConcurrentModificationException and deadlock in case of meta anassigned, // but RegionState a blocked. @@ -773,7 +779,9 @@ public class RegionStates { " to be reassigned by ServerCrashProcedure for " + sn); rits.add(hri); } else if(state.isSplittingNew() || state.isMergingNew()) { - regionsToCleanIfNoMetaEntry.add(state.getRegion()); + LOG.info("Offline/Cleanup region if no meta entry exists, hri: " + hri + + " state: " + state); + regionsToClean.add(state.getRegion()); } else { LOG.warn("THIS SHOULD NOT HAPPEN: unexpected " + state); } @@ -786,28 +794,41 @@ public class RegionStates { regionOffline(hri); } - cleanIfNoMetaEntry(regionsToCleanIfNoMetaEntry); + cleanupSplittingMergingNew(regionsToClean); return rits; } /** * This method does an RPC to hbase:meta. Do not call this method with a lock/synchronize held. + * In ZK mode we rollback and hence cleanup daughters/merged region. We also cleanup if + * meta doesn't have these regions. + * * @param hris The hris to check if empty in hbase:meta and if so, clean them up. */ - private void cleanIfNoMetaEntry(Set hris) { - if (hris.isEmpty()) return; - for (HRegionInfo hri: hris) { + private void cleanupSplittingMergingNew(Set hris) { + if (hris.isEmpty()) { + return; + } + + for (HRegionInfo hri : hris) { + // This is RPC to meta table. It is done while we have a synchronize on + // regionstates. No progress will be made if meta is not available at this time. + // This is a cleanup task. Not critical. try { - // This is RPC to meta table. It is done while we have a synchronize on - // regionstates. No progress will be made if meta is not available at this time. - // This is a cleanup task. Not critical. - if (MetaTableAccessor.getRegion(server.getConnection(), hri.getEncodedNameAsBytes()) == - null) { + Pair regionPair = + MetaTableAccessor.getRegion(server.getConnection(), hri.getRegionName()); + if (regionPair == null || useZK) { regionOffline(hri); + + // If we use ZK, then we can cleanup entries from meta, since we roll back. + if (regionPair != null) { + MetaTableAccessor.deleteRegion(this.server.getConnection(), hri); + } + LOG.debug("Cleaning up HDFS since no meta entry exists, hri: " + hri); FSUtils.deleteRegionDir(server.getConfiguration(), hri); } } catch (IOException e) { - LOG.warn("Got exception while deleting " + hri + " directories from file system.", e); + LOG.warn("Got exception while cleaning up region " + hri, e); } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java index 3eb0a558c6..59251eb701 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.regionserver; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -30,6 +31,7 @@ import java.util.List; import org.apache.commons.lang.math.RandomUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -37,12 +39,13 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.RegionTransition; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.UnknownRegionException; import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionReplicaUtil; @@ -51,17 +54,23 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.exceptions.MergeRegionException; +import org.apache.hadoop.hbase.executor.EventType; import org.apache.hadoop.hbase.master.AssignmentManager; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.master.RegionStates; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.PairOfSameType; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.util.StringUtils; +import org.apache.zookeeper.data.Stat; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -95,11 +104,13 @@ public class TestRegionMergeTransactionOnCluster { private static HMaster master; private static Admin admin; + static MiniHBaseCluster cluster; + static Configuration conf; static void setupOnce() throws Exception { // Start a cluster TEST_UTIL.startMiniCluster(NB_SERVERS); - MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + cluster = TEST_UTIL.getHBaseCluster(); master = cluster.getMaster(); master.balanceSwitch(false); admin = TEST_UTIL.getHBaseAdmin(); @@ -107,8 +118,9 @@ public class TestRegionMergeTransactionOnCluster { @BeforeClass public static void beforeAllTests() throws Exception { + conf = TEST_UTIL.getConfiguration(); // Use ZK for region assignment - TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true); + conf.setBoolean("hbase.assignment.usezk", true); setupOnce(); } @@ -469,4 +481,169 @@ public class TestRegionMergeTransactionOnCluster { assertEquals(expectedRegionNum, rowCount); scanner.close(); } + + /** + * A test that intentionally has master fail the processing of the merge message. + * Tests that the regionserver merge ephemeral node gets cleaned up if it + * crashes and that after we process server shutdown, the parent regions are online and + * merged region is cleaned up. + */ + @Test (timeout = 60000) + public void testMergeIsRolledBackOnMERGEFailure() throws Exception { + + final RegionStates regionStates = master.getAssignmentManager().getRegionStates(); + final ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher(); + + final TableName tableName = TableName.valueOf("testMergeIsRolledBackOnMERGEFailure"); + // Create table with 2 regions as its easy for us to merge. + createTableAndLoadData(master, tableName, 2, 1); + List regions = cluster.getRegions(tableName); + + assertEquals("Table shudn't have more than 2 regions, " + regions, 2, regions.size()); + final HRegionInfo regionA = regions.get(0).getRegionInfo(); + final HRegionInfo regionB = regions.get(1).getRegionInfo(); + + // Turn off balancer so it doesn't cut in and mess up our placements. + admin.setBalancerRunning(false, true); + // Turn off the meta scanner so it don't remove parent on us. + master.setCatalogJanitorEnabled(false); + + // Start a server and move both the regions to it. We kill this server later. + HRegionServer regionServer = cluster.startRegionServer().getRegionServer(); + moveRegionToServer(regionA, regionServer); + moveRegionToServer(regionB, regionServer); + + int serverIndex = cluster.getServerWith(regionA.getRegionName()); + + // This helps with server aborts later. + TEST_UTIL.compact(tableName, true); + + try { + printOutRegions(regionServer, "Initial regions: "); + + // Now, before we merge, set special flag in master, a flag that has + // it FAIL the processing of merge. + AssignmentManager.TEST_SKIP_MERGE_HANDLING = true; + admin.mergeRegions(regionA.getRegionName(), regionB.getRegionName(), false); + + // Lets wait until we have a merge region. + TEST_UTIL.waitFor(60000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return regionStates.getRegionByStateOfTable(tableName).get(State.MERGING_NEW).size() > 0; + } + }); + + List mergedRegions = + regionStates.getRegionByStateOfTable(tableName).get(State.MERGING_NEW); + assertEquals("Only one region should be in MERGING_NEW state", 1, mergedRegions.size()); + final HRegionInfo merge = mergedRegions.get(0); + + // Lets double check if we have the merge Znode with the appr. state. + final String path = ZKAssign.getNodeName(zkw, merge.getEncodedName()); + // Wait till the znode moved to MERGED + TEST_UTIL.waitFor(60000, new Waiter.Predicate() { + @Override public boolean evaluate() throws Exception { + Stat stats = zkw.getRecoverableZooKeeper().exists(path, false); + RegionTransition rt = + RegionTransition.parseFrom(ZKAssign.getData(zkw, merge.getEncodedName())); + return stats != null && rt.getEventType().equals(EventType.RS_ZK_REGION_MERGED); + } + }); + + // Now crash the server + abortServerAndWaitForProcessingToComplete(serverIndex); + waitUntilRegionServerDead(); + + TEST_UTIL.waitUntilNoRegionsInTransition(); + + // Lets wait until merge parents are online. + TEST_UTIL.waitFor(60000, new Waiter.Predicate() { + @Override + public boolean evaluate() { + return cluster.getRegions(tableName).size() == 2; + } + }); + + // Check if merge regions is cleaned up. + List tableRegions = MetaTableAccessor.getTableRegions(zkw, + cluster.getMaster().getConnection(), tableName); + assertEquals("Only parent regions should be present, but we have: " + tableRegions, + 2, tableRegions.size()); + assertTrue("Merge A not present? " + regionA, tableRegions.contains(regionA)); + assertTrue("Merge B not present? " + regionB, tableRegions.contains(regionB)); + + // Are both merge parents online? + assertTrue("region should be online, " + regionA, regionStates.isRegionOnline(regionA)); + assertTrue("region should be online, " + regionB, regionStates.isRegionOnline(regionB)); + + // Have HDFS dirs been cleaned up? + Path tableDir = FSUtils.getTableDir(FSUtils.getRootDir(conf), tableName); + List regionDirs = + FSUtils.getRegionDirs(cluster.getMaster().getFileSystem(), tableDir); + assertEquals("Only two region dir should be present, we have, dirs: " + regionDirs, + 2, regionDirs.size()); + + assertTrue("Region dir doesn't belong to region: " + regionA + " dir: " + regionDirs, + regionDirs.get(0).getName().endsWith(regionA.getEncodedName()) + || regionDirs.get(1).getName().endsWith(regionA.getEncodedName())); + assertTrue("Region dir doesn't belong to region: " + regionB + " dir: " + regionDirs, + regionDirs.get(0).getName().endsWith(regionB.getEncodedName()) + || regionDirs.get(1).getName().endsWith(regionB.getEncodedName())); + + // The merged Znode should have been cleaned up. + Stat stat = zkw.getRecoverableZooKeeper().exists(path, false); + assertNull("Merged znode shouldn't exist, but we have stat: " + stat, stat); + + } finally { + // Set this flag back. + AssignmentManager.TEST_SKIP_MERGE_HANDLING = false; + admin.setBalancerRunning(true, false); + master.setCatalogJanitorEnabled(true); + TEST_UTIL.deleteTable(tableName); + } + } + + private void moveRegionToServer(final HRegionInfo region, final HRegionServer rs) + throws Exception { + admin.move(region.getEncodedNameAsBytes(), rs.getServerName().toString().getBytes()); + TEST_UTIL.waitUntilNoRegionsInTransition(); + TEST_UTIL.waitFor(60000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return rs.getOnlineRegion(region.getRegionName()) != null; + } + }); + } + + private void waitUntilRegionServerDead() throws InterruptedException, IOException { + // Wait until the master processes the RS shutdown + for (int i=0; cluster.getMaster().getClusterStatus(). + getServers().size() > NB_SERVERS && i<100; i++) { + LOG.info("Waiting on server to go down"); + Thread.sleep(100); + } + assertFalse("Waited too long for RS to die", cluster.getMaster().getClusterStatus(). + getServers().size() > NB_SERVERS); + } + + private void printOutRegions(final HRegionServer hrs, final String prefix) + throws IOException { + List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); + for (HRegionInfo region: regions) { + LOG.info(prefix + region.getRegionNameAsString()); + } + } + + private void abortServerAndWaitForProcessingToComplete(int serverIndex) throws Exception { + + final HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); + cluster.abortRegionServer(serverIndex); + TEST_UTIL.waitFor(60000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return master.getServerManager().areDeadServersInProgress(); + } + }); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java index daeba9aba3..f83ac8163a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java @@ -126,6 +126,7 @@ public class TestSplitTransactionOnCluster { LogFactory.getLog(TestSplitTransactionOnCluster.class); private HBaseAdmin admin = null; private MiniHBaseCluster cluster = null; + private static Configuration conf; private static final int NB_SERVERS = 3; private static CountDownLatch latch = new CountDownLatch(1); private static volatile boolean secondSplit = false; @@ -143,9 +144,11 @@ public class TestSplitTransactionOnCluster { TESTING_UTIL.startMiniCluster(NB_SERVERS); } - @BeforeClass public static void before() throws Exception { + @BeforeClass + public static void before() throws Exception { + conf = TESTING_UTIL.getConfiguration(); // Use ZK for region assignment - TESTING_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true); + conf.setBoolean("hbase.assignment.usezk", true); setupOnce(); } @@ -389,39 +392,33 @@ public class TestSplitTransactionOnCluster { } } - /** + /* * A test that intentionally has master fail the processing of the split message. * Tests that the regionserver split ephemeral node gets cleaned up if it - * crashes and that after we process server shutdown, the daughters are up on - * line. - * @throws IOException - * @throws InterruptedException - * @throws NodeExistsException - * @throws KeeperException - * @throws DeserializationException + * crashes and that after we process server shutdown, the parent region is online and + * daughters are cleaned up. */ - @Test (timeout = 300000) public void testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling() - throws IOException, InterruptedException, NodeExistsException, KeeperException, - DeserializationException, ServiceException { - final TableName tableName = - TableName.valueOf("testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling"); + @Test (timeout = 60000) + public void testSplitIsRolledBackOnSPLITFailure() throws Exception { + final TableName tableName = TableName.valueOf("testSplitIsRolledBackOnSPLITFailure"); // Create table then get the single region for our new table. HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY); List regions = cluster.getRegions(tableName); - HRegionInfo hri = getAndCheckSingleTableRegion(regions); - - int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri); + final HRegionInfo hri = getAndCheckSingleTableRegion(regions); // Turn off balancer so it doesn't cut in and mess up our placements. this.admin.setBalancerRunning(false, true); // Turn off the meta scanner so it don't remove parent on us. cluster.getMaster().setCatalogJanitorEnabled(false); + + int serverIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri); + try { // Add a bit of load up into the table so splittable. TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false); // Get region pre-split. - HRegionServer server = cluster.getRegionServer(tableRegionIndex); + HRegionServer server = cluster.getRegionServer(serverIndex); printOutRegions(server, "Initial regions: "); int regionCount = ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size(); // Now, before we split, set special flag in master, a flag that has @@ -430,49 +427,60 @@ public class TestSplitTransactionOnCluster { // Now try splitting and it should work. split(hri, server, regionCount); - String path = ZKAssign.getNodeName(TESTING_UTIL.getZooKeeperWatcher(), - hri.getEncodedName()); + ZooKeeperWatcher zkw = TESTING_UTIL.getZooKeeperWatcher(); + String path = ZKAssign.getNodeName(zkw, hri.getEncodedName()); RegionTransition rt = null; Stat stats = null; - List daughters = null; - if (useZKForAssignment) { - daughters = checkAndGetDaughters(tableName); - - // Wait till the znode moved to SPLIT - for (int i=0; i<100; i++) { - stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false); - rt = RegionTransition.parseFrom(ZKAssign.getData(TESTING_UTIL.getZooKeeperWatcher(), - hri.getEncodedName())); - if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) break; - Thread.sleep(100); + + // Wait till the znode moved to SPLIT + for (int i = 0; i < 100; i++) { + stats = zkw.getRecoverableZooKeeper().exists(path, false); + rt = RegionTransition.parseFrom(ZKAssign.getData(zkw, hri.getEncodedName())); + if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) { + break; } - LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats); - assertTrue(rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)); - // Now crash the server, for ZK-less assignment, the server is auto aborted - cluster.abortRegionServer(tableRegionIndex); + Thread.sleep(100); } + LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats); + assertTrue(rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)); + // Now crash the server, for ZK-less assignment, the server is auto aborted + abortServerAndWaitForProcessingToComplete(serverIndex); waitUntilRegionServerDead(); - awaitDaughters(tableName, 2); - if (useZKForAssignment) { - regions = cluster.getRegions(tableName); - for (HRegion r: regions) { - assertTrue(daughters.contains(r)); - } - // Finally assert that the ephemeral SPLIT znode was cleaned up. - for (int i=0; i<100; i++) { - // wait a bit (10s max) for the node to disappear - stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false); - if (stats == null) break; - Thread.sleep(100); + TESTING_UTIL.waitUntilNoRegionsInTransition(); + + // Lets wait until parent region is online. + TESTING_UTIL.waitFor(60000, new Waiter.Predicate() { + @Override + public boolean evaluate() { + for (HRegion region : cluster.getRegions(tableName)) { + if (Bytes.equals(region.getRegionInfo().getRegionName(), hri.getRegionName())) { + return true; + } else { + LOG.debug("Wait for some more time, online region: " + region); + } + } + return false; } - LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats=" + stats); - assertTrue(stats == null); - } + }); + + RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates(); + assertTrue("Parent region should be online", regionStates.isRegionOnline(hri)); + // Check if daughter regions are cleaned up. + List tableRegions = MetaTableAccessor.getTableRegions(zkw, + cluster.getMaster().getConnection(), tableName); + assertEquals("Only parent region should be present, but we have: " + tableRegions, + 1, tableRegions.size()); + Path tableDir = FSUtils.getTableDir(FSUtils.getRootDir(conf), tableName); + List regionDirs = + FSUtils.getRegionDirs(cluster.getMaster().getFileSystem(), tableDir); + assertEquals("Only one region dir should be present, we have, dirs: " + regionDirs, + 1, regionDirs.size()); + assertTrue("Region dir doesn't belong to region: " + hri + " dir: " + regionDirs, + regionDirs.get(0).getName().endsWith(hri.getEncodedName())); } finally { // Set this flag back. AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false; - cluster.getMaster().getAssignmentManager().regionOffline(hri); admin.setBalancerRunning(true, false); cluster.getMaster().setCatalogJanitorEnabled(true); cluster.startRegionServer(); @@ -481,6 +489,18 @@ public class TestSplitTransactionOnCluster { } } + private void abortServerAndWaitForProcessingToComplete(int serverIndex) throws Exception { + + final HMaster master = TESTING_UTIL.getMiniHBaseCluster().getMaster(); + cluster.abortRegionServer(serverIndex); + TESTING_UTIL.waitFor(60000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return master.getServerManager().areDeadServersInProgress(); + } + }); + } + @Test (timeout = 300000) public void testExistingZnodeBlocksSplitAndWeRollback() throws IOException, InterruptedException, NodeExistsException, KeeperException, ServiceException { final TableName tableName = -- 2.14.3 (Apple Git-98)