Index: hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestFavoredNodesRepairChore.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestFavoredNodesRepairChore.java (revision ) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestFavoredNodesRepairChore.java (revision ) @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CoordinatedStateManager; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.MiniHBaseCluster.MiniHBaseClusterRegionServer; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.favored.FavoredNodesManager; +import org.apache.hadoop.hbase.master.HMasterCommandLine.LocalHMaster; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.net.ScriptBasedMapping; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Lists; + +@Category(MediumTests.class) +public class TestFavoredNodesRepairChore { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final int SLAVES = 6; + private static final int REGION_NUM = 10; + private static HBaseAdmin admin; + + @Before + public void startCluster() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + FavoredNodeLoadBalancer.class, LoadBalancer.class); + // conf.setBoolean(FavoredNodesPromoter.ALWAYS_ASSIGN_REGIONS, false); + //Don't let chore run, we will run manually when needed. + conf.setInt(FavoredNodesRepairChore.FAVORED_NODE_REPAIR_CHORE_FREQ, Integer.MAX_VALUE); + conf.set(DFSConfigKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, + ScriptBasedMapping.class.getName()); + TEST_UTIL.startMiniCluster(1, SLAVES, SLAVES, null, LocalHMaster.class, FNRegionServer.class); + TEST_UTIL.getDFSCluster().waitClusterUp(); + admin = TEST_UTIL.getHBaseAdmin(); + admin.setBalancerRunning(false, true); + } + + @After + public void stopCluster() throws Exception { + TEST_UTIL.cleanupTestDir(); + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testFavoredNodesInfoSync() throws Exception { + final TableName tableName = TableName.valueOf("testFavoredNodesInfoSync"); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM); + TEST_UTIL.waitFor(60000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return admin.isTableAvailable(tableName); + } + }); + List regionsOfTable = admin.getTableRegions(tableName); + assertEquals(REGION_NUM, regionsOfTable.size()); + final HRegionInfo candidate = regionsOfTable.get(0); + FavoredNodesManager fnm = TEST_UTIL.getMiniHBaseCluster().getMaster().getFavoredNodesManager(); + List favNodes = Lists.newArrayList(fnm.getFavoredNodes(candidate)); + final ServerName currentRS = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionServerOfRegion(candidate); + assertTrue("Current rs not part of favored nodes", + favNodes.remove(ServerName.valueOf(currentRS.getHostAndPort(), -1))); + List onlineServers = Lists.newArrayList(admin.getClusterStatus().getServers()); + assertTrue(onlineServers.remove(currentRS)); + FavoredNodeAssignmentHelper helper = new FavoredNodeAssignmentHelper(onlineServers, + TEST_UTIL.getConfiguration()); + helper.initialize(); + favNodes.add(ServerName.valueOf(helper.generateMissingFavoredNode(favNodes).getHostAndPort(), + ServerName.NON_STARTCODE)); + assertEquals(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, favNodes.size()); + fnm.updateFavoredNodes(candidate, favNodes); + FavoredNodesRepairChore chore = new FavoredNodesRepairChore(TEST_UTIL.getHBaseCluster() + .getMaster()); + chore.syncFavoredNodesWithRegionServers(); + HRegionServer regionServer = getRegionServer(currentRS); + assertTrue(regionServer instanceof FNRegionServer); + List fnFromRS = ((FNRegionServer)regionServer).getFavoredNodes(candidate.getEncodedName()); + assertNotNull(fnFromRS); + assertEquals(favNodes, fnFromRS); + } + + private HRegionServer getRegionServer(ServerName sn) { + for (RegionServerThread thread : TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()) { + if (ServerName.isSameHostnameAndPort(thread.getRegionServer().getServerName(), sn)) { + return thread.getRegionServer(); + } + } + return null; + } + + static class FNRegionServer extends MiniHBaseClusterRegionServer { + Map> fnMap = new HashMap>(); + + public FNRegionServer(Configuration conf, CoordinatedStateManager cp) + throws IOException, InterruptedException { + super(conf, cp); + } + + List getFavoredNodes(String encodedRegionName) { + return fnMap.get(encodedRegionName); + } + + @Override + public void updateRegionFavoredNodesMapping(String encodedRegionName, + List favoredNodes) { + List fns = Lists.newArrayList(); + for (org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ServerName sn : favoredNodes) { + fns.add(ProtobufUtil.toServerName(sn)); + } + fnMap.put(encodedRegionName, fns); + super.updateRegionFavoredNodesMapping(encodedRegionName, favoredNodes); + } + } +} Index: hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (date 1477605507000) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (revision ) @@ -92,6 +92,7 @@ import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer; import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore; import org.apache.hadoop.hbase.master.balancer.FavoredNodesPromoter; +import org.apache.hadoop.hbase.master.balancer.FavoredNodesRepairChore; import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory; import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; import org.apache.hadoop.hbase.master.cleaner.LogCleaner; @@ -311,6 +312,7 @@ private ClusterStatusChore clusterStatusChore; private ClusterStatusPublisher clusterStatusPublisherChore = null; private PeriodicDoMetrics periodicDoMetricsChore = null; + private FavoredNodesRepairChore favoredNodesRepairChore = null; CatalogJanitor catalogJanitorChore; private ReplicationMetaCleaner replicationMetaCleaner; @@ -803,6 +805,8 @@ getChoreService().scheduleChore(normalizerChore); this.catalogJanitorChore = new CatalogJanitor(this); getChoreService().scheduleChore(catalogJanitorChore); + favoredNodesRepairChore = new FavoredNodesRepairChore(this); + getChoreService().scheduleChore(favoredNodesRepairChore); // Do Metrics periodically periodicDoMetricsChore = new PeriodicDoMetrics(msgInterval, this); @@ -1102,6 +1106,9 @@ if (this.periodicDoMetricsChore != null) { periodicDoMetricsChore.cancel(); + } + if (this.favoredNodesRepairChore != null) { + favoredNodesRepairChore.cancel(); } } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesRepairChore.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesRepairChore.java (revision ) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesRepairChore.java (revision ) @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ScheduledChore; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.favored.FavoredNodesManager; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; + +@InterfaceAudience.Private +public class FavoredNodesRepairChore extends ScheduledChore { + + private static final Log LOG = LogFactory.getLog(FavoredNodesRepairChore.class); + + public static final String FAVORED_NODE_REPAIR_CHORE_FREQ = "hbase.favorednodes.repairPeriod"; + private final HMaster master; + private FavoredNodesManager favoredNodesManager; + + public FavoredNodesRepairChore(HMaster master) { + super(master.getServerName() + "-FavoredNodesRepairChore", master, + master.getConfiguration().getInt(FAVORED_NODE_REPAIR_CHORE_FREQ, 1 * 60 * 60000)); + this.master = master; + favoredNodesManager = master.getFavoredNodesManager(); + } + + @Override + protected void chore() { + LOG.info("Started to sync favored nodes between master and regionservers."); + long startTime = EnvironmentEdgeManager.currentTime(); + syncFavoredNodesWithRegionServers(); + LOG.info("Finished syncing favored nodes, took " + + (EnvironmentEdgeManager.currentTime() - startTime) + " ms to finish."); + } + + void syncFavoredNodesWithRegionServers() { + List onlineServers = master.getServerManager().getOnlineServersList(); + for (ServerName sn : onlineServers) { + Set regionsOfServer = master.getAssignmentManager().getRegionStates() + .getServerRegions(sn); + // Some region servers might be empty/just starting, lets ignore them + if (regionsOfServer != null && regionsOfServer.size() > 0) { + syncFavoredNodesForRS(sn, regionsOfServer); + } + } + } + + private void syncFavoredNodesForRS(ServerName server, Collection regions) { + Map> favoredNodesMap = new HashMap<>(); + for (HRegionInfo hri : regions) { + if (!hri.getTable().isSystemTable()) { + favoredNodesMap.put(hri, favoredNodesManager.getFavoredNodes(hri)); + } + } + try { + master.getServerManager().sendFavoredNodes(server, favoredNodesMap); + } catch (IOException e) { + LOG.warn("Exception while updating favored nodes on server " + server, e); + } + } +}