commit 93aa13c70386b2b319db88ae49373cea021ffd7d Author: stack Date: Tue Sep 29 23:25:59 2015 -0700 rm diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestFavoredNodeAssignmentHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestFavoredNodeAssignmentHelper.java deleted file mode 100644 index 4dc7d32..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestFavoredNodeAssignmentHelper.java +++ /dev/null @@ -1,363 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.master.balancer; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.master.RackManager; -import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Triple; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.mockito.Mockito; - -@Category({MasterTests.class, SmallTests.class}) -public class TestFavoredNodeAssignmentHelper { - - private static List servers = new ArrayList(); - private static Map> rackToServers = new HashMap>(); - private static RackManager rackManager = Mockito.mock(RackManager.class); - - @BeforeClass - public static void setupBeforeClass() throws Exception { - // Set up some server -> rack mappings - // Have three racks in the cluster with 10 hosts each. - for (int i = 0; i < 40; i++) { - ServerName server = ServerName.valueOf("foo" + i + ":1234", -1); - if (i < 10) { - Mockito.when(rackManager.getRack(server)).thenReturn("rack1"); - if (rackToServers.get("rack1") == null) { - List servers = new ArrayList(); - rackToServers.put("rack1", servers); - } - rackToServers.get("rack1").add(server); - } - if (i >= 10 && i < 20) { - Mockito.when(rackManager.getRack(server)).thenReturn("rack2"); - if (rackToServers.get("rack2") == null) { - List servers = new ArrayList(); - rackToServers.put("rack2", servers); - } - rackToServers.get("rack2").add(server); - } - if (i >= 20 && i < 30) { - Mockito.when(rackManager.getRack(server)).thenReturn("rack3"); - if (rackToServers.get("rack3") == null) { - List servers = new ArrayList(); - rackToServers.put("rack3", servers); - } - rackToServers.get("rack3").add(server); - } - servers.add(server); - } - } - - // The tests decide which racks to work with, and how many machines to - // work with from any given rack - // Return a rondom 'count' number of servers from 'rack' - private static List getServersFromRack(Map rackToServerCount) { - List chosenServers = new ArrayList(); - for (Map.Entry entry : rackToServerCount.entrySet()) { - List servers = rackToServers.get(entry.getKey()); - for (int i = 0; i < entry.getValue(); i++) { - chosenServers.add(servers.get(i)); - } - } - return chosenServers; - } - - @Test - public void testSmallCluster() { - // Test the case where we cannot assign favored nodes (because the number - // of nodes in the cluster is too less) - Map rackToServerCount = new HashMap(); - rackToServerCount.put("rack1", 2); - List servers = getServersFromRack(rackToServerCount); - FavoredNodeAssignmentHelper helper = new FavoredNodeAssignmentHelper(servers, - new Configuration()); - assertFalse(helper.canPlaceFavoredNodes()); - } - - @Test - public void testPlacePrimaryRSAsRoundRobin() { - // Test the regular case where there are many servers in different racks - // Test once for few regions and once for many regions - primaryRSPlacement(6, null, 10, 10, 10); - // now create lots of regions and try to place them on the limited number of machines - primaryRSPlacement(600, null, 10, 10, 10); - } - - @Test - public void testRoundRobinAssignmentsWithUnevenSizedRacks() { - //In the case of uneven racks, the regions should be distributed - //proportionately to the rack sizes - primaryRSPlacement(6, null, 10, 10, 10); - primaryRSPlacement(600, null, 10, 10, 5); - primaryRSPlacement(600, null, 10, 5, 10); - primaryRSPlacement(600, null, 5, 10, 10); - primaryRSPlacement(500, null, 10, 10, 5); - primaryRSPlacement(500, null, 10, 5, 10); - primaryRSPlacement(500, null, 5, 10, 10); - primaryRSPlacement(500, null, 9, 7, 8); - primaryRSPlacement(500, null, 8, 7, 9); - primaryRSPlacement(500, null, 7, 9, 8); - primaryRSPlacement(459, null, 7, 9, 8); - } - - @Test - public void testSecondaryAndTertiaryPlacementWithSingleRack() { - // Test the case where there is a single rack and we need to choose - // Primary/Secondary/Tertiary from a single rack. - Map rackToServerCount = new HashMap(); - rackToServerCount.put("rack1", 10); - // have lots of regions to test with - Triple, FavoredNodeAssignmentHelper, List> - primaryRSMapAndHelper = secondaryAndTertiaryRSPlacementHelper(60000, rackToServerCount); - FavoredNodeAssignmentHelper helper = primaryRSMapAndHelper.getSecond(); - Map primaryRSMap = primaryRSMapAndHelper.getFirst(); - List regions = primaryRSMapAndHelper.getThird(); - Map secondaryAndTertiaryMap = - helper.placeSecondaryAndTertiaryRS(primaryRSMap); - // although we created lots of regions we should have no overlap on the - // primary/secondary/tertiary for any given region - for (HRegionInfo region : regions) { - ServerName[] secondaryAndTertiaryServers = secondaryAndTertiaryMap.get(region); - assertTrue(!secondaryAndTertiaryServers[0].equals(primaryRSMap.get(region))); - assertTrue(!secondaryAndTertiaryServers[1].equals(primaryRSMap.get(region))); - assertTrue(!secondaryAndTertiaryServers[0].equals(secondaryAndTertiaryServers[1])); - } - } - - @Test - public void testSecondaryAndTertiaryPlacementWithSingleServer() { - // Test the case where we have a single node in the cluster. In this case - // the primary can be assigned but the secondary/tertiary would be null - Map rackToServerCount = new HashMap(); - rackToServerCount.put("rack1", 1); - Triple, FavoredNodeAssignmentHelper, List> - primaryRSMapAndHelper = secondaryAndTertiaryRSPlacementHelper(1, rackToServerCount); - FavoredNodeAssignmentHelper helper = primaryRSMapAndHelper.getSecond(); - Map primaryRSMap = primaryRSMapAndHelper.getFirst(); - List regions = primaryRSMapAndHelper.getThird(); - - Map secondaryAndTertiaryMap = - helper.placeSecondaryAndTertiaryRS(primaryRSMap); - // no secondary/tertiary placement in case of a single RegionServer - assertTrue(secondaryAndTertiaryMap.get(regions.get(0)) == null); - } - - @Test - public void testSecondaryAndTertiaryPlacementWithMultipleRacks() { - // Test the case where we have multiple racks and the region servers - // belong to multiple racks - Map rackToServerCount = new HashMap(); - rackToServerCount.put("rack1", 10); - rackToServerCount.put("rack2", 10); - - Triple, FavoredNodeAssignmentHelper, List> - primaryRSMapAndHelper = secondaryAndTertiaryRSPlacementHelper(60000, rackToServerCount); - FavoredNodeAssignmentHelper helper = primaryRSMapAndHelper.getSecond(); - Map primaryRSMap = primaryRSMapAndHelper.getFirst(); - - assertTrue(primaryRSMap.size() == 60000); - Map secondaryAndTertiaryMap = - helper.placeSecondaryAndTertiaryRS(primaryRSMap); - assertTrue(secondaryAndTertiaryMap.size() == 60000); - // for every region, the primary should be on one rack and the secondary/tertiary - // on another (we create a lot of regions just to increase probability of failure) - for (Map.Entry entry : secondaryAndTertiaryMap.entrySet()) { - ServerName[] allServersForRegion = entry.getValue(); - String primaryRSRack = rackManager.getRack(primaryRSMap.get(entry.getKey())); - String secondaryRSRack = rackManager.getRack(allServersForRegion[0]); - String tertiaryRSRack = rackManager.getRack(allServersForRegion[1]); - assertTrue(!primaryRSRack.equals(secondaryRSRack)); - assertTrue(secondaryRSRack.equals(tertiaryRSRack)); - } - } - - @Test - public void testSecondaryAndTertiaryPlacementWithLessThanTwoServersInRacks() { - // Test the case where we have two racks but with less than two servers in each - // We will not have enough machines to select secondary/tertiary - Map rackToServerCount = new HashMap(); - rackToServerCount.put("rack1", 1); - rackToServerCount.put("rack2", 1); - Triple, FavoredNodeAssignmentHelper, List> - primaryRSMapAndHelper = secondaryAndTertiaryRSPlacementHelper(6, rackToServerCount); - FavoredNodeAssignmentHelper helper = primaryRSMapAndHelper.getSecond(); - Map primaryRSMap = primaryRSMapAndHelper.getFirst(); - List regions = primaryRSMapAndHelper.getThird(); - assertTrue(primaryRSMap.size() == 6); - Map secondaryAndTertiaryMap = - helper.placeSecondaryAndTertiaryRS(primaryRSMap); - for (HRegionInfo region : regions) { - // not enough secondary/tertiary room to place the regions - assertTrue(secondaryAndTertiaryMap.get(region) == null); - } - } - - @Test - public void testSecondaryAndTertiaryPlacementWithMoreThanOneServerInPrimaryRack() { - // Test the case where there is only one server in one rack and another rack - // has more servers. We try to choose secondary/tertiary on different - // racks than what the primary is on. But if the other rack doesn't have - // enough nodes to have both secondary/tertiary RSs, the tertiary is placed - // on the same rack as the primary server is on - Map rackToServerCount = new HashMap(); - rackToServerCount.put("rack1", 2); - rackToServerCount.put("rack2", 1); - Triple, FavoredNodeAssignmentHelper, List> - primaryRSMapAndHelper = secondaryAndTertiaryRSPlacementHelper(6, rackToServerCount); - FavoredNodeAssignmentHelper helper = primaryRSMapAndHelper.getSecond(); - Map primaryRSMap = primaryRSMapAndHelper.getFirst(); - List regions = primaryRSMapAndHelper.getThird(); - assertTrue(primaryRSMap.size() == 6); - Map secondaryAndTertiaryMap = - helper.placeSecondaryAndTertiaryRS(primaryRSMap); - for (HRegionInfo region : regions) { - ServerName s = primaryRSMap.get(region); - ServerName secondaryRS = secondaryAndTertiaryMap.get(region)[0]; - ServerName tertiaryRS = secondaryAndTertiaryMap.get(region)[1]; - if (rackManager.getRack(s).equals("rack1")) { - assertTrue(rackManager.getRack(secondaryRS).equals("rack2") && - rackManager.getRack(tertiaryRS).equals("rack1")); - } - if (rackManager.getRack(s).equals("rack2")) { - assertTrue(rackManager.getRack(secondaryRS).equals("rack1") && - rackManager.getRack(tertiaryRS).equals("rack1")); - } - } - } - - private Triple, FavoredNodeAssignmentHelper, List> - secondaryAndTertiaryRSPlacementHelper( - int regionCount, Map rackToServerCount) { - Map primaryRSMap = new HashMap(); - List servers = getServersFromRack(rackToServerCount); - FavoredNodeAssignmentHelper helper = new FavoredNodeAssignmentHelper(servers, rackManager); - Map> assignmentMap = - new HashMap>(); - helper.initialize(); - // create regions - List regions = new ArrayList(regionCount); - for (int i = 0; i < regionCount; i++) { - HRegionInfo region = new HRegionInfo(TableName.valueOf("foobar"), - Bytes.toBytes(i), Bytes.toBytes(i + 1)); - regions.add(region); - } - // place the regions - helper.placePrimaryRSAsRoundRobin(assignmentMap, primaryRSMap, regions); - return new Triple, FavoredNodeAssignmentHelper, List> - (primaryRSMap, helper, regions); - } - - private void primaryRSPlacement(int regionCount, Map primaryRSMap, - int firstRackSize, int secondRackSize, int thirdRackSize) { - Map rackToServerCount = new HashMap(); - rackToServerCount.put("rack1", firstRackSize); - rackToServerCount.put("rack2", secondRackSize); - rackToServerCount.put("rack3", thirdRackSize); - List servers = getServersFromRack(rackToServerCount); - FavoredNodeAssignmentHelper helper = new FavoredNodeAssignmentHelper(servers, - rackManager); - helper.initialize(); - - assertTrue(helper.canPlaceFavoredNodes()); - - Map> assignmentMap = - new HashMap>(); - if (primaryRSMap == null) primaryRSMap = new HashMap(); - // create some regions - List regions = new ArrayList(regionCount); - for (int i = 0; i < regionCount; i++) { - HRegionInfo region = new HRegionInfo(TableName.valueOf("foobar"), - Bytes.toBytes(i), Bytes.toBytes(i + 1)); - regions.add(region); - } - // place those regions in primary RSs - helper.placePrimaryRSAsRoundRobin(assignmentMap, primaryRSMap, regions); - - // we should have all the regions nicely spread across the racks - int regionsOnRack1 = 0; - int regionsOnRack2 = 0; - int regionsOnRack3 = 0; - for (HRegionInfo region : regions) { - if (rackManager.getRack(primaryRSMap.get(region)).equals("rack1")) { - regionsOnRack1++; - } else if (rackManager.getRack(primaryRSMap.get(region)).equals("rack2")) { - regionsOnRack2++; - } else if (rackManager.getRack(primaryRSMap.get(region)).equals("rack3")) { - regionsOnRack3++; - } - } - // Verify that the regions got placed in the way we expect (documented in - // FavoredNodeAssignmentHelper#placePrimaryRSAsRoundRobin) - checkNumRegions(regionCount, firstRackSize, secondRackSize, thirdRackSize, regionsOnRack1, - regionsOnRack2, regionsOnRack3, assignmentMap); - } - - private void checkNumRegions(int regionCount, int firstRackSize, int secondRackSize, - int thirdRackSize, int regionsOnRack1, int regionsOnRack2, int regionsOnRack3, - Map> assignmentMap) { - //The regions should be distributed proportionately to the racksizes - //Verify the ordering was as expected by inserting the racks and regions - //in sorted maps. The keys being the racksize and numregions; values are - //the relative positions of the racksizes and numregions respectively - SortedMap rackMap = new TreeMap(); - rackMap.put(firstRackSize, 1); - rackMap.put(secondRackSize, 2); - rackMap.put(thirdRackSize, 3); - SortedMap regionMap = new TreeMap(); - regionMap.put(regionsOnRack1, 1); - regionMap.put(regionsOnRack2, 2); - regionMap.put(regionsOnRack3, 3); - assertTrue(printProportions(firstRackSize, secondRackSize, thirdRackSize, - regionsOnRack1, regionsOnRack2, regionsOnRack3), - rackMap.get(firstRackSize) == regionMap.get(regionsOnRack1)); - assertTrue(printProportions(firstRackSize, secondRackSize, thirdRackSize, - regionsOnRack1, regionsOnRack2, regionsOnRack3), - rackMap.get(secondRackSize) == regionMap.get(regionsOnRack2)); - assertTrue(printProportions(firstRackSize, secondRackSize, thirdRackSize, - regionsOnRack1, regionsOnRack2, regionsOnRack3), - rackMap.get(thirdRackSize) == regionMap.get(regionsOnRack3)); - } - - private String printProportions(int firstRackSize, int secondRackSize, - int thirdRackSize, int regionsOnRack1, int regionsOnRack2, int regionsOnRack3) { - return "The rack sizes " + firstRackSize + " " + secondRackSize - + " " + thirdRackSize + " " + regionsOnRack1 + " " + regionsOnRack2 + - " " + regionsOnRack3; - } -}