From 1085c3d7969db7d1b5fe049e20c4914f083d09e4 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Sun, 6 Oct 2019 19:58:46 +0800 Subject: [PATCH] HBASE-23125 TestRSGroupsAdmin2 is flaky --- .../hbase/rsgroup/TestRSGroupsAdmin2.java | 28 +++++----- .../hbase/rsgroup/TestRSGroupsBase.java | 51 +++++++------------ 2 files changed, 34 insertions(+), 45 deletions(-) diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java index 6553a85c93..4b4097fee7 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java @@ -618,14 +618,15 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase { } private Pair randomlySetRegionState(RSGroupInfo groupInfo, - RegionState.State state, TableName... tableNames) throws IOException { + RegionState.State state, TableName... tableNames) throws IOException { Preconditions.checkArgument(tableNames.length == 1 || tableNames.length == 2, - "only support one or two tables"); - Map> assignMap = getTableServerRegionMap().get(tableNames[0]); - if(tableNames.length == 2) { - Map> assignMap2 = getTableServerRegionMap().get(tableNames[1]); - assignMap2.forEach((k ,v) -> { - if(!assignMap.containsKey(k)) { + "only support one or two tables"); + Map>> tableServerRegionMap = getTableServerRegionMap(); + Map> assignMap = tableServerRegionMap.get(tableNames[0]); + if (tableNames.length == 2) { + Map> assignMap2 = tableServerRegionMap.get(tableNames[1]); + assignMap2.forEach((k, v) -> { + if (!assignMap.containsKey(k)) { assignMap.remove(k); } }); @@ -633,8 +634,10 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase { String toCorrectRegionName = null; ServerName srcServer = null; for (ServerName server : assignMap.keySet()) { - toCorrectRegionName = assignMap.get(server).size() >= 1 && - !groupInfo.containsServer(server.getAddress()) ? assignMap.get(server).get(0) : null; + toCorrectRegionName = + assignMap.get(server).size() >= 1 && !groupInfo.containsServer(server.getAddress()) + ? assignMap.get(server).get(0) + : null; if (toCorrectRegionName != null) { srcServer = server; break; @@ -642,10 +645,9 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase { } assert srcServer != null; RegionInfo toCorrectRegionInfo = TEST_UTIL.getMiniHBaseCluster().getMaster() - .getAssignmentManager().getRegionInfo(Bytes.toBytesBinary(toCorrectRegionName)); - RegionStateNode rsn = - TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates() - .getRegionStateNode(toCorrectRegionInfo); + .getAssignmentManager().getRegionInfo(Bytes.toBytesBinary(toCorrectRegionName)); + RegionStateNode rsn = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionStateNode(toCorrectRegionInfo); rsn.setState(state); return new Pair<>(srcServer, rsn); } diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java index c5520cf11f..0e4fb34189 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.rsgroup; import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.util.ArrayList; import java.util.EnumSet; import java.util.HashSet; import java.util.LinkedList; @@ -37,8 +38,6 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.NamespaceDescriptor; -import org.apache.hadoop.hbase.RegionMetrics; -import org.apache.hadoop.hbase.ServerMetrics; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; @@ -55,7 +54,6 @@ import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.ServerManager; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.net.Address; -import org.apache.hadoop.hbase.util.Bytes; import org.junit.Rule; import org.junit.rules.TestName; import org.slf4j.Logger; @@ -185,7 +183,7 @@ public abstract class TestRSGroupsBase { }); } - public RSGroupInfo addGroup(String groupName, int serverCount) + protected RSGroupInfo addGroup(String groupName, int serverCount) throws IOException, InterruptedException { RSGroupInfo defaultInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP); rsGroupAdmin.addRSGroup(groupName); @@ -201,21 +199,21 @@ public abstract class TestRSGroupsBase { return result; } - public void removeGroup(String groupName) throws IOException { + protected void removeGroup(String groupName) throws IOException { RSGroupInfo groupInfo = rsGroupAdmin.getRSGroupInfo(groupName); rsGroupAdmin.moveTables(groupInfo.getTables(), RSGroupInfo.DEFAULT_GROUP); rsGroupAdmin.moveServers(groupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP); rsGroupAdmin.removeRSGroup(groupName); } - public void deleteTableIfNecessary() throws IOException { + protected void deleteTableIfNecessary() throws IOException { for (TableDescriptor desc : TEST_UTIL.getAdmin() .listTableDescriptors(Pattern.compile(tablePrefix + ".*"))) { TEST_UTIL.deleteTable(desc.getTableName()); } } - public void deleteNamespaceIfNecessary() throws IOException { + protected void deleteNamespaceIfNecessary() throws IOException { for (NamespaceDescriptor desc : TEST_UTIL.getAdmin().listNamespaceDescriptors()) { if(desc.getName().startsWith(tablePrefix)) { admin.deleteNamespace(desc.getName()); @@ -223,7 +221,7 @@ public abstract class TestRSGroupsBase { } } - public void deleteGroups() throws IOException { + protected void deleteGroups() throws IOException { RSGroupAdmin groupAdmin = new RSGroupAdminClient(TEST_UTIL.getConnection()); for(RSGroupInfo group: groupAdmin.listRSGroups()) { if(!group.getName().equals(RSGroupInfo.DEFAULT_GROUP)) { @@ -234,7 +232,7 @@ public abstract class TestRSGroupsBase { } } - public Map> getTableRegionMap() throws IOException { + protected Map> getTableRegionMap() throws IOException { Map> map = Maps.newTreeMap(); Map>> tableServerRegionMap = getTableServerRegionMap(); @@ -249,35 +247,24 @@ public abstract class TestRSGroupsBase { return map; } - public Map>> getTableServerRegionMap() - throws IOException { + protected Map>> getTableServerRegionMap() + throws IOException { Map>> map = Maps.newTreeMap(); - ClusterMetrics status = TEST_UTIL.getHBaseClusterInterface().getClusterMetrics(); - for (Map.Entry entry : status.getLiveServerMetrics().entrySet()) { - ServerName serverName = entry.getKey(); - for(RegionMetrics rl : entry.getValue().getRegionMetrics().values()) { - TableName tableName = null; - try { - tableName = RegionInfo.getTable(rl.getRegionName()); - } catch (IllegalArgumentException e) { - LOG.warn("Failed parse a table name from regionname=" + - Bytes.toStringBinary(rl.getRegionName())); - continue; - } - if(!map.containsKey(tableName)) { - map.put(tableName, new TreeMap<>()); - } - if(!map.get(tableName).containsKey(serverName)) { - map.get(tableName).put(serverName, new LinkedList<>()); - } - map.get(tableName).get(serverName).add(rl.getNameAsString()); + Admin admin = TEST_UTIL.getAdmin(); + ClusterMetrics metrics = + admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.SERVERS_NAME)); + for (ServerName serverName : metrics.getServersName()) { + for (RegionInfo region : admin.getRegions(serverName)) { + TableName tableName = region.getTable(); + map.computeIfAbsent(tableName, k -> new TreeMap<>()) + .computeIfAbsent(serverName, k -> new ArrayList<>()).add(region.getRegionNameAsString()); } } return map; } // return the real number of region servers, excluding the master embedded region server in 2.0+ - public int getNumServers() throws IOException { + protected int getNumServers() throws IOException { ClusterMetrics status = admin.getClusterMetrics(EnumSet.of(Option.MASTER, Option.LIVE_SERVERS)); ServerName masterName = status.getMasterName(); @@ -290,7 +277,7 @@ public abstract class TestRSGroupsBase { return count; } - public String getGroupName(String baseName) { + protected String getGroupName(String baseName) { return groupPrefix + "_" + baseName + "_" + rand.nextInt(Integer.MAX_VALUE); } -- 2.17.1